#Modified from https://github.com/matterport/Mask_RCNN

import sys
import os
import math
import random
import numpy as np
import tensorflow as tf
import scipy.misc
import skimage.color
import skimage.io
from skimage.util import crop
from skimage.color import label2rgb
from skimage.io import imread, imshow, imread_collection, concatenate_images
from skimage.morphology import convex_hull
from scipy import ndimage as ndi
import urllib.request
import shutil
from tqdm import tqdm

# URL from which to download the latest COCO trained weights
COCO_MODEL_URL = "https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5"

############################################################
#  Bounding Boxes
############################################################

def extract_bboxes(mask):
    """Compute bounding boxes from masks.
    mask: [height, width, num_instances]. Mask pixels are either 1 or 0.

    Returns: bbox array [num_instances, (y1, x1, y2, x2)].
    """
    boxes = np.zeros([mask.shape[-1], 4], dtype=np.int32)
    for i in range(mask.shape[-1]):
        m = mask[:, :, i]
        # Bounding box.
        horizontal_indicies = np.where(np.any(m, axis=0))[0]
        vertical_indicies = np.where(np.any(m, axis=1))[0]
        if horizontal_indicies.shape[0]:
            x1, x2 = horizontal_indicies[[0, -1]]
            y1, y2 = vertical_indicies[[0, -1]]
            # x2 and y2 should not be part of the box. Increment by 1.
            x2 += 1
            y2 += 1
        else:
            # No mask for this instance. Might happen due to
            # resizing or cropping. Set bbox to zeros
            x1, x2, y1, y2 = 0, 0, 0, 0
        boxes[i] = np.array([y1, x1, y2, x2])
    return boxes.astype(np.int32)

def compute_iou(box, boxes, box_area, boxes_area):
    """Calculates IoU of the given box with the array of the given boxes.
    box: 1D vector [y1, x1, y2, x2]
    boxes: [boxes_count, (y1, x1, y2, x2)]
    box_area: float. the area of 'box'
    boxes_area: array of length boxes_count.

    Note: the areas are passed in rather than calculated here for
          efficency. Calculate once in the caller to avoid duplicate work.
    """
    # Calculate intersection areas
    y1 = np.maximum(box[0], boxes[:, 0])
    y2 = np.minimum(box[2], boxes[:, 2])
    x1 = np.maximum(box[1], boxes[:, 1])
    x2 = np.minimum(box[3], boxes[:, 3])
    intersection = np.maximum(x2 - x1, 0) * np.maximum(y2 - y1, 0)
    union = box_area + boxes_area[:] - intersection[:]
    iou = intersection / union
    return iou

def compute_overlaps(boxes1, boxes2):
    """Computes IoU overlaps between two sets of boxes.
    boxes1, boxes2: [N, (y1, x1, y2, x2)].

    For better performance, pass the largest set first and the smaller second.
    """
    # Areas of anchors and GT boxes
    area1 = (boxes1[:, 2] - boxes1[:, 0]) * (boxes1[:, 3] - boxes1[:, 1])
    area2 = (boxes2[:, 2] - boxes2[:, 0]) * (boxes2[:, 3] - boxes2[:, 1])

    # Compute overlaps to generate matrix [boxes1 count, boxes2 count]
    # Each cell contains the IoU value.
    overlaps = np.zeros((boxes1.shape[0], boxes2.shape[0]))
    for i in range(overlaps.shape[1]):
        box2 = boxes2[i]
        overlaps[:, i] = compute_iou(box2, boxes1, area2[i], area1)
    return overlaps

def compute_overlaps_masks(masks1, masks2):
    '''Computes IoU overlaps between two sets of masks.
    masks1, masks2: [Height, Width, instances]
    '''
    # flatten masks
    masks1 = np.reshape(masks1 > .5, (-1, masks1.shape[-1])).astype(np.float32)
    masks2 = np.reshape(masks2 > .5, (-1, masks2.shape[-1])).astype(np.float32)
    area1 = np.sum(masks1, axis=0)
    area2 = np.sum(masks2, axis=0)

    # intersections and union
    intersections = np.dot(masks1.T, masks2)
    union = area1[:, None] + area2[None, :] - intersections
    overlaps = intersections / union

    return overlaps

def non_max_suppression(boxes, scores, threshold):
    """Performs non-maximum supression and returns indicies of kept boxes.
    boxes: [N, (y1, x1, y2, x2)]. Notice that (y2, x2) lays outside the box.
    scores: 1-D array of box scores.
    threshold: Float. IoU threshold to use for filtering.
    """
    assert boxes.shape[0] > 0
    if boxes.dtype.kind != "f":
        boxes = boxes.astype(np.float32)

    # Compute box areas
    y1 = boxes[:, 0]
    x1 = boxes[:, 1]
    y2 = boxes[:, 2]
    x2 = boxes[:, 3]
    area = (y2 - y1) * (x2 - x1)

    # Get indicies of boxes sorted by scores (highest first)
    ixs = scores.argsort()[::-1]

    pick = []
    while len(ixs) > 0:
        # Pick top box and add its index to the list
        i = ixs[0]
        pick.append(i)
        # Compute IoU of the picked box with the rest
        iou = compute_iou(boxes[i], boxes[ixs[1:]], area[i], area[ixs[1:]])
        # Identify boxes with IoU over the threshold. This
        # returns indicies into ixs[1:], so add 1 to get
        # indicies into ixs.
        remove_ixs = np.where(iou > threshold)[0] + 1
        # Remove indicies of the picked and overlapped boxes.
        ixs = np.delete(ixs, remove_ixs)
        ixs = np.delete(ixs, 0)
    return np.array(pick, dtype=np.int32)

def apply_box_deltas(boxes, deltas):
    """Applies the given deltas to the given boxes.
    boxes: [N, (y1, x1, y2, x2)]. Note that (y2, x2) is outside the box.
    deltas: [N, (dy, dx, log(dh), log(dw))]
    """
    boxes = boxes.astype(np.float32)
    # Convert to y, x, h, w
    height = boxes[:, 2] - boxes[:, 0]
    width = boxes[:, 3] - boxes[:, 1]
    center_y = boxes[:, 0] + 0.5 * height
    center_x = boxes[:, 1] + 0.5 * width
    # Apply deltas
    center_y += deltas[:, 0] * height
    center_x += deltas[:, 1] * width
    height *= np.exp(deltas[:, 2])
    width *= np.exp(deltas[:, 3])
    # Convert back to y1, x1, y2, x2
    y1 = center_y - 0.5 * height
    x1 = center_x - 0.5 * width
    y2 = y1 + height
    x2 = x1 + width
    return np.stack([y1, x1, y2, x2], axis=1)

def box_refinement_graph(box, gt_box):
    """Compute refinement needed to transform box to gt_box.
    box and gt_box are [N, (y1, x1, y2, x2)]
    """
    box = tf.cast(box, tf.float32)
    gt_box = tf.cast(gt_box, tf.float32)

    height = box[:, 2] - box[:, 0]
    width = box[:, 3] - box[:, 1]
    center_y = box[:, 0] + 0.5 * height
    center_x = box[:, 1] + 0.5 * width

    gt_height = gt_box[:, 2] - gt_box[:, 0]
    gt_width = gt_box[:, 3] - gt_box[:, 1]
    gt_center_y = gt_box[:, 0] + 0.5 * gt_height
    gt_center_x = gt_box[:, 1] + 0.5 * gt_width

    dy = (gt_center_y - center_y) / height
    dx = (gt_center_x - center_x) / width
    dh = tf.log(gt_height / height)
    dw = tf.log(gt_width / width)

    result = tf.stack([dy, dx, dh, dw], axis=1)
    return result

def box_refinement(box, gt_box):
    """Compute refinement needed to transform box to gt_box.
    box and gt_box are [N, (y1, x1, y2, x2)]. (y2, x2) is
    assumed to be outside the box.
    """
    box = box.astype(np.float32)
    gt_box = gt_box.astype(np.float32)

    height = box[:, 2] - box[:, 0]
    width = box[:, 3] - box[:, 1]
    center_y = box[:, 0] + 0.5 * height
    center_x = box[:, 1] + 0.5 * width

    gt_height = gt_box[:, 2] - gt_box[:, 0]
    gt_width = gt_box[:, 3] - gt_box[:, 1]
    gt_center_y = gt_box[:, 0] + 0.5 * gt_height
    gt_center_x = gt_box[:, 1] + 0.5 * gt_width

    dy = (gt_center_y - center_y) / height
    dx = (gt_center_x - center_x) / width
    dh = np.log(gt_height / height)
    dw = np.log(gt_width / width)

    return np.stack([dy, dx, dh, dw], axis=1)

############################################################
#  Dataset
############################################################

class Dataset(object):
    """The base class for dataset classes.
    To use it, create a new class that adds functions specific to the dataset
    you want to use. For example:

    class CatsAndDogsDataset(Dataset):
        def load_cats_and_dogs(self):
            ...
        def load_mask(self, image_id):
            ...
        def image_reference(self, image_id):
            ...

    See COCODataset and ShapesDataset as examples.
    """

    def __init__(self, class_map=None):
        self._image_ids = []
        self.image_info = []
        # Background is always the first class
        self.class_info = [{"source": "", "id": 0, "name": "BG"}]
        self.source_class_ids = {}

    def add_class(self, source, class_id, class_name):
        assert "." not in source, "Source name cannot contain a dot"
        # Does the class exist already?
        for info in self.class_info:
            if info['source'] == source and info["id"] == class_id:
                # source.class_id combination already available, skip
                return
        # Add the class
        self.class_info.append({
            "source": source,
            "id": class_id,
            "name": class_name,
        })

    def add_image(self, source, image_id, path, **kwargs):
        image_info = {
            "id": image_id,
            "source": source,
            "path": path,
        }
        image_info.update(kwargs)
        self.image_info.append(image_info)

    def image_reference(self, image_id):
        """Return a link to the image in its source Website or details about
        the image that help looking it up or debugging it.

        Override for your dataset, but pass to this function
        if you encounter images not in your dataset.
        """
        return ""

    def prepare(self, class_map=None):
        """Prepares the Dataset class for use.

        TODO: class map is not supported yet. When done, it should handle mapping
              classes from different datasets to the same class ID.
        """

        def clean_name(name):
            """Returns a shorter version of object names for cleaner display."""
            return ",".join(name.split(",")[:1])

        # Build (or rebuild) everything else from the info dicts.
        self.num_classes = len(self.class_info)
        self.class_ids = np.arange(self.num_classes)
        self.class_names = [clean_name(c["name"]) for c in self.class_info]
        self.num_images = len(self.image_info)
        self._image_ids = np.arange(self.num_images)

        self.class_from_source_map = {"{}.{}".format(info['source'], info['id']): id
                                      for info, id in zip(self.class_info, self.class_ids)}

        # Map sources to class_ids they support
        self.sources = list(set([i['source'] for i in self.class_info]))
        self.source_class_ids = {}
        # Loop over datasets
        for source in self.sources:
            self.source_class_ids[source] = []
            # Find classes that belong to this dataset
            for i, info in enumerate(self.class_info):
                # Include BG class in all datasets
                if i == 0 or source == info['source']:
                    self.source_class_ids[source].append(i)

    def map_source_class_id(self, source_class_id):
        """Takes a source class ID and returns the int class ID assigned to it.

        For example:
        dataset.map_source_class_id("coco.12") -> 23
        """
        return self.class_from_source_map[source_class_id]

    def get_source_class_id(self, class_id, source):
        """Map an internal class ID to the corresponding class ID in the source dataset."""
        info = self.class_info[class_id]
        assert info['source'] == source
        return info['id']

    def append_data(self, class_info, image_info):
        self.external_to_class_id = {}
        for i, c in enumerate(self.class_info):
            for ds, id in c["map"]:
                self.external_to_class_id[ds + str(id)] = i

        # Map external image IDs to internal ones.
        self.external_to_image_id = {}
        for i, info in enumerate(self.image_info):
            self.external_to_image_id[info["ds"] + str(info["id"])] = i

    @property
    def image_ids(self):
        return self._image_ids

    def source_image_link(self, image_id):
        """Returns the path or URL to the image.
        Override this to return a URL to the image if it's availble online for easy
        debugging.
        """
        return self.image_info[image_id]["path"]

    def load_image(self, image_id):
        """Load the specified image and return a [H,W,3] Numpy array.
        """
        # Load image
        image = skimage.io.imread(self.image_info[image_id]['path'])
        # If grayscale. Convert to RGB for consistency.
        if image.ndim != 3:
            image = skimage.color.gray2rgb(image)
        return image

    def load_mask(self, image_id):
        """Load instance masks for the given image.

        Different datasets use different ways to store masks. Override this
        method to load instance masks and return them in the form of am
        array of binary masks of shape [height, width, instances].

        Returns:
            masks: A bool array of shape [height, width, instance count] with
                a binary mask per instance.
            class_ids: a 1D array of class IDs of the instance masks.
        """
        # Override this function to load a mask from your dataset.
        # Otherwise, it returns an empty mask.
        mask = np.empty([0, 0, 0])
        class_ids = np.empty([0], np.int32)
        return mask, class_ids

def resize_image(image, min_dim=None, max_dim=None, padding=False):
    """
    Resizes an image keeping the aspect ratio.

    min_dim: if provided, resizes the image such that it's smaller
        dimension == min_dim
    max_dim: if provided, ensures that the image longest side doesn't
        exceed this value.
    padding: If true, pads image with zeros so it's size is max_dim x max_dim

    Returns:
    image: the resized image
    window: (y1, x1, y2, x2). If max_dim is provided, padding might
        be inserted in the returned image. If so, this window is the
        coordinates of the image part of the full image (excluding
        the padding). The x2, y2 pixels are not included.
    scale: The scale factor used to resize the image
    padding: Padding added to the image [(top, bottom), (left, right), (0, 0)]
    """
    # Default window (y1, x1, y2, x2) and default scale == 1.
    h, w = image.shape[:2]
    window = (0, 0, h, w)
    scale = 1

    # Scale?
    if min_dim:
        # Scale up but not down
        scale = max(1, min_dim / min(h, w))
    # Does it exceed max dim?
    if max_dim:
        image_max = max(h, w)
        if round(image_max * scale) > max_dim:
            scale = max_dim / image_max
    # Resize image and mask
    if scale != 1:
        image = scipy.misc.imresize(
            image, (round(h * scale), round(w * scale)))
    # Need padding?
    if padding:
        # Get new height and width
        h, w = image.shape[:2]
        top_pad = (max_dim - h) // 2
        bottom_pad = max_dim - h - top_pad
        left_pad = (max_dim - w) // 2
        right_pad = max_dim - w - left_pad
        padding = [(top_pad, bottom_pad), (left_pad, right_pad), (0, 0)]
        image = np.pad(image, padding, mode='constant', constant_values=0)
        window = (top_pad, left_pad, h + top_pad, w + left_pad)
    return image, window, scale, padding

def resize_mask(mask, scale, padding):
    """Resizes a mask using the given scale and padding.
    Typically, you get the scale and padding from resize_image() to
    ensure both, the image and the mask, are resized consistently.

    scale: mask scaling factor
    padding: Padding to add to the mask in the form
            [(top, bottom), (left, right), (0, 0)]
    """
    h, w = mask.shape[:2]
    mask = scipy.ndimage.zoom(mask, zoom=[scale, scale, 1], order=0)
    mask = np.pad(mask, padding, mode='constant', constant_values=0)
    return mask

def minimize_mask(bbox, mask, mini_shape):
    """Resize masks to a smaller version to cut memory load.
    Mini-masks can then resized back to image scale using expand_masks()

    See inspect_data.ipynb notebook for more details.
    """
    mini_mask = np.zeros(mini_shape + (mask.shape[-1],), dtype=bool)
    for i in range(mask.shape[-1]):
        m = mask[:, :, i]
        y1, x1, y2, x2 = bbox[i][:4]
        m = m[y1:y2, x1:x2]
        if m.size == 0:
            raise Exception("Invalid bounding box with area of zero")
        m = scipy.misc.imresize(m.astype(float), mini_shape, interp='bilinear')
        mini_mask[:, :, i] = np.where(m >= 128, 1, 0)
    return mini_mask

def expand_mask(bbox, mini_mask, image_shape):
    """Resizes mini masks back to image size. Reverses the change
    of minimize_mask().

    See inspect_data.ipynb notebook for more details.
    """
    mask = np.zeros(image_shape[:2] + (mini_mask.shape[-1],), dtype=bool)
    for i in range(mask.shape[-1]):
        m = mini_mask[:, :, i]
        y1, x1, y2, x2 = bbox[i][:4]
        h = y2 - y1
        w = x2 - x1
        m = scipy.misc.imresize(m.astype(float), (h, w), interp='bilinear')
        mask[y1:y2, x1:x2, i] = np.where(m >= 128, 1, 0)
    return mask

# TODO: Build and use this function to reduce code duplication
def mold_mask(mask, config):
    pass

def unmold_mask(mask, bbox, image_shape):
    """Converts a mask generated by the neural network into a format similar
    to it's original shape.
    mask: [height, width] of type float. A small, typically 28x28 mask.
    bbox: [y1, x1, y2, x2]. The box to fit the mask in.

    Returns a binary mask with the same size as the original image.
    """
    threshold = 0.5
    y1, x1, y2, x2 = bbox
    mask = scipy.misc.imresize(
        mask, (y2 - y1, x2 - x1), interp='bilinear').astype(np.float32) / 255.0
    mask = np.where(mask >= threshold, 1, 0).astype(np.uint8)

    # Put the mask in the right location.
    full_mask = np.zeros(image_shape[:2], dtype=np.uint8)
    full_mask[y1:y2, x1:x2] = mask
    return full_mask

############################################################
#  Anchors
############################################################

def generate_anchors(scales, ratios, shape, feature_stride, anchor_stride):
    """
    scales: 1D array of anchor sizes in pixels. Example: [32, 64, 128]
    ratios: 1D array of anchor ratios of width/height. Example: [0.5, 1, 2]
    shape: [height, width] spatial shape of the feature map over which
            to generate anchors.
    feature_stride: Stride of the feature map relative to the image in pixels.
    anchor_stride: Stride of anchors on the feature map. For example, if the
        value is 2 then generate anchors for every other feature map pixel.
    """
    # Get all combinations of scales and ratios
    scales, ratios = np.meshgrid(np.array(scales), np.array(ratios))
    scales = scales.flatten()
    ratios = ratios.flatten()

    # Enumerate heights and widths from scales and ratios
    heights = scales / np.sqrt(ratios)
    widths = scales * np.sqrt(ratios)

    # Enumerate shifts in feature space
    shifts_y = np.arange(0, shape[0], anchor_stride) * feature_stride
    shifts_x = np.arange(0, shape[1], anchor_stride) * feature_stride
    shifts_x, shifts_y = np.meshgrid(shifts_x, shifts_y)

    # Enumerate combinations of shifts, widths, and heights
    box_widths, box_centers_x = np.meshgrid(widths, shifts_x)
    box_heights, box_centers_y = np.meshgrid(heights, shifts_y)

    # Reshape to get a list of (y, x) and a list of (h, w)
    box_centers = np.stack(
        [box_centers_y, box_centers_x], axis=2).reshape([-1, 2])
    box_sizes = np.stack([box_heights, box_widths], axis=2).reshape([-1, 2])

    # Convert to corner coordinates (y1, x1, y2, x2)
    boxes = np.concatenate([box_centers - 0.5 * box_sizes,
                            box_centers + 0.5 * box_sizes], axis=1)
    return boxes

def generate_pyramid_anchors(scales, ratios, feature_shapes, feature_strides,
                             anchor_stride):
    """Generate anchors at different levels of a feature pyramid. Each scale
    is associated with a level of the pyramid, but each ratio is used in
    all levels of the pyramid.

    Returns:
    anchors: [N, (y1, x1, y2, x2)]. All generated anchors in one array. Sorted
        with the same order of the given scales. So, anchors of scale[0] come
        first, then anchors of scale[1], and so on.
    """
    # Anchors
    # [anchor_count, (y1, x1, y2, x2)]
    anchors = []
    for i in range(len(scales)):
        anchors.append(generate_anchors(scales[i], ratios, feature_shapes[i],
                                        feature_strides[i], anchor_stride))
    return np.concatenate(anchors, axis=0)

############################################################
#  Miscellaneous
############################################################

def trim_zeros(x):
    """It's common to have tensors larger than the available data and
    pad with zeros. This function removes rows that are all zeros.

    x: [rows, columns].
    """
    assert len(x.shape) == 2
    return x[~np.all(x == 0, axis=1)]

def masks2labels(masks):
    num_mask = masks.shape[2]
    rolled = np.rollaxis(masks, 2)
    mask_list = list(rolled)
    mask_list.sort(key=lambda x: np.sum(x), reverse=False)
    #print( [np.sum(x) for x in mask_list])
    labels = np.zeros((masks.shape[0], masks.shape[1]), dtype=np.uint16)
    for i in range(num_mask):
        mask_list[i] = np.where(mask_list[i] > 0, 1, 0)
        mask_list[i] = ndi.morphology.binary_fill_holes(mask_list[i])
        #mask_list[i] = convex_hull.convex_hull_image(mask_list[i])
        labels = np.where(mask_list[i] > 0, i + 1, labels)
    return labels

def label_loss(label_true, label_pred):
    true_objects = len(np.unique(label_true))
    pred_objects = len(np.unique(label_pred))

    intersection = np.histogram2d(label_true.flatten(), label_pred.flatten(), bins=(true_objects, pred_objects))[0]

    area_true = np.histogram(label_true, bins = true_objects)[0]
    area_pred = np.histogram(label_pred, bins = pred_objects)[0]
    area_true = np.expand_dims(area_true, -1)
    area_pred = np.expand_dims(area_pred, 0)

    union = area_true + area_pred - intersection

    intersection = intersection[1:,1:]
    union = union[1:,1:]
    union[union == 0] = 1e-9

    iou = intersection / union

    def precision_at(threshold, iou):
        matches = iou > threshold
        true_positives = np.sum(matches, axis=1) == 1   # Correct objects
        false_positives = np.sum(matches, axis=0) == 0  # Missed objects
        false_negatives = np.sum(matches, axis=1) == 0  # Extra objects
        tp, fp, fn = np.sum(true_positives), np.sum(false_positives), np.sum(false_negatives)
        return tp, fp, fn

    prec = []

    for t in np.arange(0.5, 1.0, 0.05):
        tp, fp, fn = precision_at(t, iou)
        p = tp / (tp + fp + fn)
        prec.append(p)
    return np.mean(prec)

def compute_map_nuclei(true_masks, pred_masks):
    true_labels = masks2labels(true_masks)
    pred_labels = masks2labels(pred_masks)
    return label_loss(true_labels, pred_labels)

def compute_ap(gt_boxes, gt_class_ids, gt_masks,
               pred_boxes, pred_class_ids, pred_scores, pred_masks,
               iou_threshold=0.5):
    """Compute Average Precision at a set IoU threshold (default 0.5).

    Returns:
    mAP: Mean Average Precision
    precisions: List of precisions at different class score thresholds.
    recalls: List of recall values at different class score thresholds.
    overlaps: [pred_boxes, gt_boxes] IoU overlaps.
    """
    # Trim zero padding and sort predictions by score from high to low
    # TODO: cleaner to do zero unpadding upstream
    gt_boxes = trim_zeros(gt_boxes)
    gt_masks = gt_masks[..., :gt_boxes.shape[0]]
    pred_boxes = trim_zeros(pred_boxes)
    pred_scores = pred_scores[:pred_boxes.shape[0]]
    indices = np.argsort(pred_scores)[::-1]
    pred_boxes = pred_boxes[indices]
    pred_class_ids = pred_class_ids[indices]
    pred_scores = pred_scores[indices]
    pred_masks = pred_masks[..., indices]

    # Compute IoU overlaps [pred_masks, gt_masks]
    overlaps = compute_overlaps_masks(pred_masks, gt_masks)

    # Loop through ground truth boxes and find matching predictions
    match_count = 0
    pred_match = np.zeros([pred_boxes.shape[0]])
    gt_match = np.zeros([gt_boxes.shape[0]])
    for i in range(len(pred_boxes)):
        # Find best matching ground truth box
        sorted_ixs = np.argsort(overlaps[i])[::-1]
        for j in sorted_ixs:
            # If ground truth box is already matched, go to next one
            if gt_match[j] == 1:
                continue
            # If we reach IoU smaller than the threshold, end the loop
            iou = overlaps[i, j]
            if iou < iou_threshold:
                break
            # Do we have a match?
            if pred_class_ids[i] == gt_class_ids[j]:
                match_count += 1
                gt_match[j] = 1
                pred_match[i] = 1
                break

    # Compute precision and recall at each prediction box step
    precisions = np.cumsum(pred_match) / (np.arange(len(pred_match)) + 1)
    recalls = np.cumsum(pred_match).astype(np.float32) / len(gt_match)

    # Pad with start and end values to simplify the math
    precisions = np.concatenate([[0], precisions, [0]])
    recalls = np.concatenate([[0], recalls, [1]])

    # Ensure precision values decrease but don't increase. This way, the
    # precision value at each recall threshold is the maximum it can be
    # for all following recall thresholds, as specified by the VOC paper.
    for i in range(len(precisions) - 2, -1, -1):
        precisions[i] = np.maximum(precisions[i], precisions[i + 1])

    # Compute mean AP over recall range
    indices = np.where(recalls[:-1] != recalls[1:])[0] + 1
    mAP = np.sum((recalls[indices] - recalls[indices - 1]) *
                 precisions[indices])

    return mAP, precisions, recalls, overlaps

def compute_recall(pred_boxes, gt_boxes, iou):
    """Compute the recall at the given IoU threshold. It's an indication
    of how many GT boxes were found by the given prediction boxes.

    pred_boxes: [N, (y1, x1, y2, x2)] in image coordinates
    gt_boxes: [N, (y1, x1, y2, x2)] in image coordinates
    """
    # Measure overlaps
    overlaps = compute_overlaps(pred_boxes, gt_boxes)
    iou_max = np.max(overlaps, axis=1)
    iou_argmax = np.argmax(overlaps, axis=1)
    positive_ids = np.where(iou_max >= iou)[0]
    matched_gt_boxes = iou_argmax[positive_ids]

    recall = len(set(matched_gt_boxes)) / gt_boxes.shape[0]
    return recall, positive_ids

# ## Batch Slicing
# Some custom layers support a batch size of 1 only, and require a lot of work
# to support batches greater than 1. This function slices an input tensor
# across the batch dimension and feeds batches of size 1. Effectively,
# an easy way to support batches > 1 quickly with little code modification.
# In the long run, it's more efficient to modify the code to support large
# batches and getting rid of this function. Consider this a temporary solution
def batch_slice(inputs, graph_fn, batch_size, names=None):
    """Splits inputs into slices and feeds each slice to a copy of the given
    computation graph and then combines the results. It allows you to run a
    graph on a batch of inputs even if the graph is written to support one
    instance only.

    inputs: list of tensors. All must have the same first dimension length
    graph_fn: A function that returns a TF tensor that's part of a graph.
    batch_size: number of slices to divide the data into.
    names: If provided, assigns names to the resulting tensors.
    """
    if not isinstance(inputs, list):
        inputs = [inputs]

    outputs = []
    for i in range(batch_size):
        inputs_slice = [x[i] for x in inputs]
        output_slice = graph_fn(*inputs_slice)
        if not isinstance(output_slice, (tuple, list)):
            output_slice = [output_slice]
        outputs.append(output_slice)
    # Change outputs from a list of slices where each is
    # a list of outputs to a list of outputs and each has
    # a list of slices
    outputs = list(zip(*outputs))

    if names is None:
        names = [None] * len(outputs)

    result = [tf.stack(o, axis=0, name=n)
              for o, n in zip(outputs, names)]
    if len(result) == 1:
        result = result[0]

    return result

def download_trained_weights(coco_model_path, verbose=1):
    """Download COCO trained weights from Releases.

    coco_model_path: local path of COCO trained weights
    """
    if verbose > 0:
        print("Downloading pretrained model to " + coco_model_path + " ...")
    with urllib.request.urlopen(COCO_MODEL_URL) as resp, open(coco_model_path, 'wb') as out:
        shutil.copyfileobj(resp, out)
    if verbose > 0:
        print("... done downloading pretrained model!")

class NuDataset(Dataset):

    def load_nuclei(self, image_list, masks_list):
        """Generate the requested number of synthetic images.
        count: number of images to generate.
        height, width: the size of the generated images.
        """
        # Add classes
        self.add_class("nuclei", 1, "nuclei")

        # Add images
        # Generate random specifications of images (i.e. color and
        # list of shapes sizes and locations). This is more compact than
        # actual images. Images are generated on the fly in load_image().
        for i in range(len(image_list)):
            self.add_image("nuclei", image_id=i, path=None,
                           width=image_list[i].shape[0], height=image_list[i].shape[1],
                           masks=masks_list[i], image=image_list[i])
            if image_list[i].shape[0]!=masks_list[i].shape[0] or image_list[i].shape[1]!=masks_list[i].shape[1]:
                print("!!!!!")

    def load_image(self, image_id):
        """Generate an image from the specs of the given image ID.
        Typically this function loads the image from a file, but
        in this case it generates the image on the fly from the
        specs in image_info.
        """
        info = self.image_info[image_id]
        image = info['image']
        return image

    def image_reference(self, image_id):
        """Return the shapes data of the image."""
        info = self.image_info[image_id]
        if info["source"] == "nuclei":
            return info["nuclei"]
        else:
            super(self.__class__).image_reference(self, image_id)

    def load_mask(self, image_id):
        """Generate instance masks for shapes of the given image ID.
        """
        info = self.image_info[image_id]
        masks = info['masks']
        count = masks.shape[2]

        # Map class names to class IDs.
        class_ids = np.array([1 for i in range(count)])
        return masks, class_ids.astype(np.int32)

    def load_and_augment(self, config, image_id, augment):
        image = self.load_image(image_id)
        mask, class_ids = self.load_mask(image_id)
        shape = image.shape
        image, window, scale, padding = resize_image(
            image,
            min_dim=config.IMAGE_MIN_DIM,
            max_dim=config.IMAGE_MAX_DIM,
            padding=config.IMAGE_PADDING)
        mask = resize_mask(mask, scale, padding)

        return image, mask, class_ids, window, shape

def load_train_data():
    print("Loading training data from disk")

    TRAIN_PATH = '../stage1_train_fix/'  #https://github.com/lopuhin/kaggle-dsbowl-2018-dataset-fixes

    IMG_CHANNELS = 3
    seed = 42
    random.seed(seed)
    np.random.seed = seed

    train_ids = next(os.walk(TRAIN_PATH))[1]

    # Get and resize train images and masks
    X_train = []
    Y_train = []
    print('Getting and resizing train images and masks ... ')
    sys.stdout.flush()
    for n, id_ in tqdm(enumerate(train_ids), total=len(train_ids)):
        path = TRAIN_PATH + id_
        img = imread(path + '/images/' + id_ + '.png')[:,:,:IMG_CHANNELS]

        X_train.append(img)
        mask_list = []

        index = 0
        for mask_file in next(os.walk(path + '/masks/'))[2]:
            index = index + 1
            mask_ = imread(path + '/masks/' + mask_file)
            mask__ = np.zeros_like(mask_, dtype = np.uint8)
            mask__ = np.where(mask_ > 0, 1, 0)
            mask_ = mask__

            mask_list.append(mask_)

        Y_train.append(np.array(mask_list))
        Y_train[n] = np.moveaxis(Y_train[n], 0, -1)

    import csv

    STAGE1_TEST_PATH = '../stage1_test/'
    STAGE1_SOLUTION_PATH = '../stage1_solution.csv'
    stage1_test_ids = next(os.walk(STAGE1_TEST_PATH))[1]
    stage1_test_sizes = {}
    stage1_mask_list = {}
    IMG_CHANNELS = 3

    print('Getting test images ... ')
    sys.stdout.flush()
    for n, id_ in tqdm(enumerate(stage1_test_ids), total=len(stage1_test_ids)):
        path = STAGE1_TEST_PATH + id_
        img0 = imread(path + '/images/' + id_ + '.png')
        #print(id_)
        if img0.ndim==2:
            img = np.stack((img0,)*3, -1)
        else:
            img = img0[:,:,:IMG_CHANNELS]
        X_train.append(img)
        stage1_test_sizes[id_] = [img.shape[0], img.shape[1]]
        stage1_mask_list[id_] = []

    with open('../stage1_solution.csv', newline='') as csvfile:
        reader = csv.DictReader(csvfile)
        for row in reader:
            #print(row)
            rleNumbers = [int(s) for s in row['EncodedPixels'].split(' ')]
            rlePairs = np.array(rleNumbers).reshape(-1,2)
            height = stage1_test_sizes[row['ImageId']][0]
            width = stage1_test_sizes[row['ImageId']][1]

            mask = np.zeros(height*width,dtype=np.uint8)
            for index,length in rlePairs:
                index -= 1
                mask[index:index+length] = 1
            mask = mask.reshape(width,height)
            mask = mask.T
            stage1_mask_list[row['ImageId']].append(mask)

    for n, id_ in tqdm(enumerate(stage1_test_ids), total=len(stage1_test_ids)):
        Y_train.append(np.array(stage1_mask_list[id_]))
        nn = len(Y_train) - 1
        Y_train[nn] = np.moveaxis(Y_train[nn], 0, -1)

    print('Done!')
    return X_train, Y_train

def load_test_data():
    IMG_CHANNELS = 3

    # Get and resize test images
    TEST_PATH = '../stage2_test_final/'
    test_ids = next(os.walk(TEST_PATH))[1]

    X_test = []
    sizes_test = []
    print('Getting and resizing test images ... ')
    sys.stdout.flush()
    for n, id_ in tqdm(enumerate(test_ids), total=len(test_ids)):
        path = TEST_PATH + id_
        img0 = imread(path + '/images/' + id_ + '.png')
        #print(id_)
        if img0.ndim==2:
            img = np.stack((img0,)*3, -1)
        else:
            img = img0[:,:,:IMG_CHANNELS]
        sizes_test.append([img.shape[0], img.shape[1]])
        X_test.append(img)
    return X_test, sizes_test, test_ids

def split_train_val(X_train, Y_train, ratio = 0.95, seed = 42):
    indices = [i for i in range(len(X_train))]
    random.seed(seed)
    random.shuffle(indices)

    train_size = int(len(X_train) * ratio)
    val_size = len(X_train) - train_size

    print("Validation set indices")
    print(indices[train_size:])
    print("Training set indices")
    print(indices[:train_size])

    x_val = [X_train[i] for i in indices[train_size:]]
    y_val = [Y_train[i] for i in indices[train_size:]]
    print("Validation set size")
    print(len(x_val))
    print(len(y_val))
    x_train = [X_train[i] for i in indices[:train_size]]
    y_train = [Y_train[i] for i in indices[:train_size]]
    print("Training set size")
    print(len(x_train))
    print(len(y_train))

    return x_train, y_train, x_val, y_val

# Run-length encoding with labels
def rle_encoding(x):
    dots = np.where(x.T.flatten() == 1)[0]
    run_lengths = []
    prev = -2
    for b in dots:
        if (b>prev+1): run_lengths.extend((b + 1, 0))
        run_lengths[-1] += 1
        prev = b
    return run_lengths

def prob_to_rles(x, cutoff=0.5):
    lab_img = masks2labels(x)
    for i in range(1, lab_img.max() + 1):
        result = rle_encoding(lab_img == i)
        if len(result) > 0:
            yield result


from oauth.OAuth2 import OAuth2
import json
from logger import logger

client_id = ''
client_secret = ''
redirect_uri = 'urn:ietf:wg:oauth:2.0:oob'
authorization_base_url = "https://accounts.google.com/o/oauth2/auth"
token_url = "https://accounts.google.com/o/oauth2/token"
refresh_url = 'https://www.googleapis.com/oauth2/v3/token'
scope = ['https://www.googleapis.com/auth/urlshortener']

googleOauth = OAuth2(client_id, client_secret, redirect_uri, authorization_base_url, token_url, refresh_url, scope)

def shorten(longUrl):
	session = googleOauth.getSession()
	post_url = 'https://www.googleapis.com/urlshortener/v1/url'
	payload = {'longUrl': longUrl}
	headers = {'content-type': 'application/json'}

	response = session.post(post_url, data=json.dumps(payload), headers=headers)

	response = json.loads(response.text)

	if response.get('error') == None:
		return response['id']
	else:
		logger.error('Google URL Shortener Error: {}, URL: {}'.format(response['error']['errors'], longUrl))
		return longUrl


# -*- coding: utf-8 -*-

from odoo import models, fields, api
from odoo import tools, _
from odoo.exceptions import ValidationError
import logging

_logger = logging.getLogger(__name__)
class ResPartnerSuffix(models.Model):
    _name = 'res.partner.suffix'
    _rec_name = 'name'
    _inherit = ['mail.thread']

    _mail_post_access = 'read'

    _sql_constraints = [('unique_name', 'unique(name)', 'Suffix Already exists.')]
    name = fields.Char(string="Suffix", required=True, index=True)
    shortcut = fields.Char(string="Abbreviation", required=True)

    @api.constrains('name')
    def _validate_name(self):
        id = self.id
        name = self.name
        res = self.search([['name', '=ilike', name], ['id', '!=', id]])
        if res:
            raise ValidationError(_("Suffix Name already exists."))

    @api.constrains('shortcut')
    def _validate_name(self):
        id = self.id
        shortcut = self.shortcut
        res = self.search([['shortcut', '=ilike', shortcut], ['id', '!=', id]])
        if res:
            raise ValidationError(_("Suffix Abbreviation already exists."))

class LoanClient(models.Model):
    _inherit = 'res.partner'

    display_name = fields.Char(compute='_compute_display_name', store=True, index=True)
    code = fields.Char(readonly=True)
    short_code = fields.Char(readonly=True)
    index = fields.Integer()
    type = fields.Selection(selection_add=[('member','Member'),
                             ('do','Development Officer'),
                             ('ds','Development Supervisor'),
                             ('ao','Account Officer'),
                             ('as','Account Supervisor'),
                             ('aa','Admin Assistant'),
                             ('bm','Branch Manager'),
                             ('gm','General Manager')], default='member', string='User Type')
    type_str = fields.Char(default=lambda self:self.default_type())
    branch_id = fields.Many2one('res.branch','Branch', required_if_type=['bm','aa','as','ao','ds','do'],default= lambda self: self.default_branch(), store=True)
    area_id = fields.Many2one('res.area','Area', required_if_type='do')
    area = fields.Char(related='area_id.name', string='Area')
    supervisor_id = fields.Many2one('res.partner','Supervisor', domain=[('type','in',['ds','as','aa','bm','gm'])])
    email = fields.Char(required=True)
    mobile = fields.Char(required=True)
    employees = fields.One2many('res.partner', 'parent_id', 'Employees',default=lambda self:self.child_ids, domain=[('type','!=','member')])
    members = fields.One2many('res.partner', 'parent_id', 'Members', default=lambda self:self.child_ids, domain=[('type','=','member')])

    @api.depends('name')
    def _compute_display_name(self):
        for partner in self:
            partner.display_name = partner.name

    @api.onchange('type')
    def _set_type(self):
        self.type_str = dict(self._fields['type'].selection).get(self.type)

    @api.multi
    def default_type(self):
        return dict(self._fields['type'].selection).get(self.type)

    @api.multi
    def default_branch(self):
        if self.type == 'do':
            return self.parent_id.branch_id
        else:
            return self.env['res.branch'].search([('street2','=',self.street2),('city','=', self.city)], limit=1)

    @api.onchange('street2', 'city', 'type')
    def _get_branch(self):
        if self.type == 'member':
            self.branch_id = self.env['res.branch'].search([('street2','=',self.street2),('city','=', self.city)], limit=1)
        elif self.type == 'gm':
            self.branch_id = None
        else:
            self.branch_id = self.parent_id.branch_id

    @api.model
    def create(self, values):

        if not values['is_company']:
            group = self.env['res.groups'].sudo().search([('name', '=', values.get('type_str'))], limit=1)
            values['index'] = int(self.search([('type', '=', values.get('type'))], order='index desc', limit=1).index) + 1
            values['employee'] = True
            values['customer'] = False
            if values['type'] == 'gm':
                company = self.env['res.company'].search([('id', '=', values.get('company_id'))], limit=1)

                values['code'] = '%s' % ("{0:0=2d}".format(values['index']))
                values['short_code'] = '%s' % ("{0:0=2d}".format(values['index']))
                parent = company.partner_id
                values['parent_id'] = parent.id

            else:
                branch = self.env['res.branch'].search([('id', '=', values.get('branch_id'))], limit=1)

                values['code'] = '%s-%s' % (
                "{0:0=2d}".format(branch.index),
                "{0:0=2d}".format(values['index']))
                values['short_code'] = '%s' % ("{0:0=2d}".format(values['index']))
                parent = branch.related_partner
                values['parent_id'] = parent.id

                if values['type'] == 'member':
                    values['employee'] = False
                    values['customer'] = True

            partner = super(LoanClient, self).create(values)
            user = self.env['res.users'].sudo().create({
                'partner_id': partner.id,
                'login': values.get('email'),
                'groups_id': group,
                'company_ids': [partner.parent_id.company_id.id],
                'company_id': parent.company_id.id
            })
            self.write({'user_id':user.id})

            return partner
        else:
            return super(LoanClient, self).create(values)


# -*- coding: utf-8 -*-

# Copyright © 2016 Roberto Alsina and others

# This plugin is based on the Pelican ical plugin
# available at https://github.com/getpelican/pelican-plugins/tree/master/ical
#
# This plugin is licensed under the
# GNU AFFERO GENERAL PUBLIC LICENSE Version 3
# according to the license guidelines from the Pelican plugin repo
#
# Original code is written by:
#
# Julien Ortet: https://github.com/cozo
# Justin Mayer: https://github.com/justinmayer
# calfzhou: https://github.com/calfzhou

from __future__ import print_function

import doit
from datetime import datetime, timedelta
from dateutil.rrule import rrulestr, rruleset
import icalendar as ical
from pytz import UTC

from nikola.plugin_categories import ShortcodePlugin
from nikola.utils import LocaleBorg

class CalendarPlugin(ShortcodePlugin):
    """Calendar shortcode."""
    name = "ical"

    doc_purpose = "Display ical calendars"
    doc_description = "Format and display ical calendars."
    logger = None
    cmd_options = []

    def set_site(self, site):
        super(CalendarPlugin, self).set_site(site)
        self.site.register_shortcode('calendar', self.handler)

    def handler(self, site=None, data=None, lang=None, file=None, template=None, post=None, days_in_future=None, days_in_past=0):
        if not template:
            template = 'calendar.tmpl'
        deps = self.site.template_system.template_deps(template)

        if file is not None:
            with open(file, 'rb') as inf:
                data = inf.read()
            deps.append(file)
        cal = ical.Calendar.from_ical(data)

        events = []
        for element in cal.walk():
            eventdict = {}
            if element.name == "VEVENT":
                if element.get('summary') is not None:
                    eventdict['summary'] = element.get('summary')
                if element.get('description') is not None:
                    eventdict['description'] = element.get('description')
                if element.get('url') is not None:
                    eventdict['url'] = element.get('url')
                if element.get('dtstart') is not None:
                    eventdict['dtstart'] = element.get('dtstart').dt
                if element.get('dtend') is not None:
                    eventdict['dtend'] = element.get('dtend').dt

                rules_text = '\n'.join([line for line in element.content_lines() if line.startswith('RRULE')])
                if days_in_future is not None and rules_text:

                    # Build rrule to use for calculation if available
                    rules = rruleset()
                    first_rule = rrulestr(rules_text, dtstart=element.get('dtstart').dt)

                    # force UTC if no tzinfo is present in until part (bug in older iCal and Moz)
                    if first_rule._until and first_rule._until.tzinfo is None:
                        first_rule._until = first_rule._until.replace(tzinfo=UTC)
                    rules.rrule(first_rule)

                    # Also check for excluded dates in entry, has API list bug for single entry
                    exdates = element.get('exdate')
                    if not isinstance(exdates, list):
                        exdates = [exdates]
                    for exdate in exdates:
                        #doit.tools.set_trace()
                        try:
                            #rules.exdate(exdate.dts[0].dt)
                            pass
                        except AttributeError:  # skip empty entries
                            pass

                    calc_startdate = datetime.now(tz=UTC)
                    calc_startdate -= timedelta(days=int(days_in_past))
                    calc_enddate = datetime.now(tz=UTC)
                    calc_enddate += timedelta(days=int(days_in_future))

                    for entry_calcdate in rules.between(calc_startdate, calc_enddate):
                        new_entry = eventdict.copy()
                        duration = new_entry['dtend'] - new_entry['dtstart']
                        new_entry['dtstart'] = entry_calcdate
                        new_entry['dtend'] = entry_calcdate + duration
                        events.append(new_entry)
                else:
                    events.append(eventdict)

        output = self.site.render_template(
            template,
            None,
            {
                'events': events,
                'lang': LocaleBorg().current_lang,
            })

        return output, deps


from torch import nn
import numpy as np
import torch
from torch.nn import functional as F
from models.sync_batchnorm.batchnorm import SynchronizedBatchNorm2d

class Generator(nn.Module):
    def __init__(self, BatchNorm, nz=100, nch_g=64, nch=3):
        super(Generator, self).__init__()
        self.layers = nn.ModuleDict({
            'layer0': nn.Sequential(
                nn.ConvTranspose2d(nz, nch_g * 8, 4, 1, 0, bias=False),
                BatchNorm(nch_g * 8),
                nn.LeakyReLU(0.2, inplace=True),
            ),  # (100, 1, 1) -> (512, 4, 4)
            'layer1': nn.Sequential(
                nn.ConvTranspose2d(nch_g * 8, nch_g * 4, 4, 2, 1, bias=False),
                BatchNorm(nch_g * 4),
                nn.LeakyReLU(0.2, inplace=True),
            ),  # (512, 4, 4) -> (256, 8, 8)
            'layer2': nn.Sequential(
                nn.ConvTranspose2d(nch_g * 4, nch_g * 2, 4, 2, 1, bias=False),
                BatchNorm(nch_g * 2),
                nn.LeakyReLU(0.2, inplace=True),
            ),  # (256, 8, 8) -> (128, 16, 16)

            'layer3': nn.Sequential(
                nn.ConvTranspose2d(nch_g * 2, nch_g, 4, 2, 1, bias=False),
                BatchNorm(nch_g),
                nn.LeakyReLU(0.2, inplace=True),
            ),  # (128, 16, 16) -> (64, 32, 32)
            # 'layer4': nn.Sequential(
            #     nn.ConvTranspose2d(nch_g, nch_g, 4, 2, 1),
            #     nn.Tanh()
            # )   # (64, 32, 32) -> (3, 64, 64)
        })
        self._init_weight()

    def forward(self, z):
        for layer in self.layers.values():
            z = layer(z)
        return z

    def _init_weight(self):
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                torch.nn.init.kaiming_normal_(m.weight)
            elif isinstance(m, SynchronizedBatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()
            elif isinstance(m, nn.SyncBatchNorm):
                m.weight.data.fill_(1)
                m.bias.data.zero_()

class DaSTGenerator(nn.Module):
    def __init__(self, n_classes,n_channel, BatchNorm=SynchronizedBatchNorm2d):
        super(DaSTGenerator, self).__init__()

        class_gen=[]
        for i in range(n_classes):
            class_gen.append(Generator(BatchNorm))
        self.class_gen=nn.ModuleList(class_gen)

        self.n_classes=n_classes

        self.conv=nn.Sequential(
                    nn.Conv2d(in_channels = 64,out_channels = 64,kernel_size = 3, padding =1, bias=False),
                    nn.BatchNorm2d(64),
                    # nn.ReLU(),
                    nn.LeakyReLU(0.2, inplace=True),
                    nn.Conv2d(in_channels = 64,out_channels = 128,kernel_size = 3,padding = 1, bias=False),
                    nn.BatchNorm2d(128),
                    # nn.ReLU(),
                    nn.LeakyReLU(0.2, inplace=True),
                    nn.Conv2d(in_channels = 128,out_channels = 64,kernel_size = 3, padding =1, bias=False),
                    nn.BatchNorm2d(64),
                    # nn.ReLU(),
                    nn.LeakyReLU(0.2, inplace=True),
                    nn.Conv2d(in_channels = 64,out_channels = 32,kernel_size = 3, padding =1, bias=False),
                    nn.BatchNorm2d(32),
                    # nn.ReLU(),
                    nn.LeakyReLU(0.2, inplace=True),
                    nn.Conv2d(in_channels = 32,out_channels = n_channel,kernel_size = 3, padding =1, bias=True),
                    # nn.BatchNorm2d(n_channel),
                    nn.Sigmoid()
                    )

        self._init_weight()

    def forward(self, batch_size,device='cpu',shuffle=False,size=(28,28)):

        class_bs=batch_size//self.n_classes+((batch_size%self.n_classes)>0)

        z=torch.randn(class_bs, 100, 1, 1).to(device)
        outputs=self.class_gen[0](z)
        outputs=F.interpolate(outputs, size=size, mode='bilinear', align_corners=True)
        targets=torch.Tensor([0]*class_bs).to(device)
        for i in range(1,self.n_classes):
            z=torch.randn(class_bs, 100, 1, 1).to(device)
            output=self.class_gen[i](z)
            output=F.interpolate(output, size=size, mode='bilinear', align_corners=True)
            outputs=torch.cat([outputs,output],dim=0)
            targets=torch.cat([targets,torch.Tensor([i]*class_bs).to(device)],dim=0)

        outputs=self.conv(outputs)

        N=len(outputs)
        if shuffle:
            idx=np.random.permutation(range(N))
        else:
            idx=list(range(N))

        return outputs[idx],targets[idx]

    def _init_weight(self):
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                torch.nn.init.kaiming_normal_(m.weight)
            elif isinstance(m, SynchronizedBatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()
            elif isinstance(m, nn.SyncBatchNorm):
                m.weight.data.fill_(1)
                m.bias.data.zero_()


# Databricks notebook source
import requests
import io
import os
from pyspark.sql.functions import col
from timeit import default_timer as timer
from datetime import timedelta
from torchvision import transforms
from PIL import Image

# COMMAND ----------

# OUTPUT_FILE = "/mnt/group07/final_data_product/downloaded_images-1.parquet"
OUTPUT_FILE = "/mnt/group07/final_data_product/downloaded_images-retry-final_v2.parquet"
# INPUT_FILE = "/mnt/group07/final_data_product/classification_input.parquet"
INPUT_FILE = "/mnt/group07/final_data_product/download_input-retry.parquet"

# COMMAND ----------

class ImageDownloader:
    transform_function = transforms.Compose([
        transforms.Resize(224),
        transforms.CenterCrop(224)
    ])

    @staticmethod
    def transform_image(image_bytes):
        image = Image.open(image_bytes)
        transformed_image = ImageDownloader.transform_function(image)
        return transformed_image

# COMMAND ----------

def download_and_save_image(row):
  image_bytes = None
  image_height = None
  image_width = None
  image_bytes_size = None

  if row.longitude is not None:
    try:
      img_s3_path = os.path.join("/dbfs/mnt/group01/images", "{0}_{1}_{2}.{3}".format(row.id, row.longitude, row.latitude, row.photo_video_extension_original) )
      s3_img_file = open(img_s3_path, "rb")
      img_read = s3_img_file.read()
      s3_img_file.close()
      # get image stats:
      image_bytes = bytearray(img_read)

      # resize image
      image_bytes_size = len(img_read)
      image = Image.open(io.BytesIO(image_bytes))
      image_height = image.size[1]
      image_width = image.size[0]

      img = ImageDownloader.transform_image(io.BytesIO(image_bytes))
      buffer = io.BytesIO()
      # Convert to RGB to fix error (POST FIX)
      img = img.convert('RGB')
      img.save(buffer, format='PNG')
      image_bytes = bytearray(buffer.getvalue())

      return (row.id, image_bytes, image_bytes_size, image_height, image_width, "s3")
    except Exception as e:
      # do nothing, and retry on real flickr data
      pass

  # try to download:
  try:
    req = requests.get(row.photo_video_download_url)
    if req.status_code == 200:
      image_bytes = bytearray(req.content)
      image_bytes_size = len(req.content)
      # resize image
      image = Image.open(io.BytesIO(image_bytes))
      image_height = image.size[1]
      image_width = image.size[0]

      img = ImageDownloader.transform_image(io.BytesIO(image_bytes))
      buffer = io.BytesIO()
      # Convert to RGB to fix error (POST FIX)
      img = img.convert('RGB')
      img.save(buffer, format='PNG')
      image_bytes = bytearray(buffer.getvalue())

    else:
      image_bytes = None
    return (row.id, image_bytes, image_bytes_size, image_height, image_width, req.status_code)
  except Exception as e:
    s = str(e)
    return (row.id, None, None, None, None, s)

# COMMAND ----------

# Start Timing
start_download = timer()

# COMMAND ----------

# DBTITLE 1,Whole Download Pipeline
from pyspark.sql.types import StructField, BinaryType, StructType, StringType, IntegerType

# Define Schema for output table
schema = StructType(fields=[
    StructField('id', StringType(), True),
    StructField('img_binary', BinaryType(), True),
    StructField('img_size', IntegerType(), True),
    StructField('img_width', IntegerType(), True),
    StructField('img_height', IntegerType(), True),
    StructField('error_code', StringType(), True),
  ])

df_dl_links = spark.read.parquet(INPUT_FILE)

print("About to process {0} rows".format(df_dl_links.count()))
# Sort the dataframe along id
df_dl_links = df_dl_links.sort("id")
# Load the previously downloaded images (if job restarted)
if os.path.isdir("/dbfs" + OUTPUT_FILE):
  download_history_df = spark.read.parquet(OUTPUT_FILE)
else:
  download_history_df = spark.createDataFrame([], schema=schema)

# Determine the rows that have been downloaded already + determine the rows to be downloaded again
already_downloaded_ids = download_history_df.where(col("img_binary").isNotNull()).select("id")
to_download_df = df_dl_links.join(already_downloaded_ids, 'id', 'left_anti')
already_downloaded = already_downloaded_ids.count()
if (already_downloaded > 0):
  print("Restarted from {0} rows - downloading {1} rows".format(already_downloaded, to_download_df.count()))

# COMMAND ----------

batch_map = to_download_df.rdd.map(download_and_save_image)
final_df = spark.createDataFrame(batch_map, schema=schema)

# COMMAND ----------

# Create the new download file from previous downloads + new downloads
download_history_df = download_history_df.where(col("img_binary").isNotNull()).union(final_df)

# Write results to storage
download_history_df.write.format("parquet").save(OUTPUT_FILE)

# COMMAND ----------

# Finish timer
end_download = timer()
download_in_seconds = timedelta(seconds=end_download-start_download).total_seconds()
print(download_in_seconds)

# COMMAND ----------

df_output = spark.read.parquet(OUTPUT_FILE)

# COMMAND ----------

df_output.groupBy("error_code").count().show(truncate=False)

# COMMAND ----------

df_output.groupBy("error_code").count().show(truncate=False)

# COMMAND ----------

df_output.count()

# COMMAND ----------

df_output.show()

# COMMAND ----------


import torch as pt

def get_num_splits(size: int, split_size: int) -> int:
    if size % split_size == 0:
        return size // split_size

    return (size + split_size - 1) // split_size

def get_tf_device(device: pt.device):
    if device.type == "cuda":
        name = "GPU"
        index = device.index
        if index is None:
            index = 0
    elif device.type == "cpu":
        name = "CPU"
        index = 0
    else:
        raise RuntimeError(f"Unknown device type: {device.type}!")

    return f"{name}:{index}"


# Copyright (C) 2018 Atsushi Togo
# All rights reserved.
#
# This file is part of phonopy.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
#   notice, this list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in
#   the documentation and/or other materials provided with the
#   distribution.
#
# * Neither the name of the phonopy project nor the names of its
#   contributors may be used to endorse or promote products derived
#   from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import os
import numpy as np
from phonopy.api_phonopy import Phonopy
from phonopy.structure.cells import (guess_primitive_matrix,
                                     get_primitive_matrix_by_centring)
from phonopy.interface import (read_crystal_structure,
                               get_default_physical_units)
from phonopy.file_IO import (parse_BORN, parse_FORCE_SETS,
                             read_force_constants_hdf5,
                             parse_FORCE_CONSTANTS)

def load(supercell_matrix=None,
         primitive_matrix=None,
         is_nac=False,
         calculator="vasp",
         unitcell=None,
         supercell=None,
         nac_params=None,
         unitcell_filename=None,
         supercell_filename=None,
         born_filename=None,
         force_sets_filename=None,
         force_constants_filename=None,
         use_alm=False,
         factor=None,
         frequency_scale_factor=None,
         symprec=1e-5,
         is_symmetry=True,
         log_level=0):
    """Create Phonopy instance from parameters and/or input files.

    When unitcell and unitcell_filename are not given, file name that is
    default for the chosen calculator is looked for in the current directory
    as the default behaviour.

    When force_sets_filename and force_constants_filename are not given,
    'FORCE_SETS' is looked for in the current directory as the default
    behaviour.

    Parameters
    ----------
    supercell_matrix : array_like, optional
        Supercell matrix multiplied to input cell basis vectors.
        shape=(3, ) or (3, 3), where the former is considered a diagonal
        matrix. Default is the unit matrix.
        dtype=int
    primitive_matrix : array_like or str, optional
        Primitive matrix multiplied to input cell basis vectors. Default is
        the identity matrix.
        shape=(3, 3)
        dtype=float
        When 'F', 'I', 'A', 'C', or 'R' is given instead of a 3x3 matrix,
        the primitive matrix defined at
        https://atztogo.github.io/spglib/definition.html
        is used.
    is_nac : bool, optional
        If True, look for 'BORN' file.
        The priority for NAC is nac_params > born_filename > is_nac ('BORN')
    calculator : str, optional.
        Calculator used for computing forces. This is used to switch the set
        of physical units. Default is 'vasp'.
    unitcell : PhonopyAtoms, optional
        Input unit cell. Default is None. The priority for cell is
        unitcell_filename > supercell_filename > unitcell > supercell.
    supercell : PhonopyAtoms, optional
        Input supercell cell. Default value of primitive_matrix is set to
        'auto' (can be overwitten). supercell_matrix is ignored. Default is
        None. The priority for cell is
        unitcell_filename > supercell_filename > unitcell > supercell.
    nac_params : dict, optional
        Parameters required for non-analytical term correction. Default is
        None. The priority for NAC is nac_params > born_filename > is_nac.
        {'born': Born effective charges
                 (array_like, shape=(primitive cell atoms, 3, 3), dtype=float),
         'dielectric': Dielectric constant matrix
                       (array_like, shape=(3, 3), dtype=float),
         'factor': unit conversion facotr (float)}
    unitcell_filename : str, optional
        Input unit cell filename. Default is None. The priority for cell is
        unitcell_filename > supercell_filename > unitcell > supercell.
    supercell_filename : str, optional
        Input supercell filename. Default value of primitive_matrix is set to
        'auto' (can be overwitten). supercell_matrix is ignored. Default is
        None. The priority for cell is
        unitcell_filename > supercell_filename > unitcell > supercell.
    born_filename : str, optional
        Filename corresponding to 'BORN', a file contains non-analytical term
        correction parameters.
        The priority for NAC is nac_params > born_filename > is_nac ('BORN').
    force_sets_filename : str, optional
        Filename of a file corresponding to 'FORCE_SETS', a file contains sets
        of forces and displacements. Default is None.
        The priority for force constants is
        force_constants_filename > force_sets_filename > 'FORCE_SETS'.
    force_constants_filename : str, optional
        Filename of a file corresponding to 'FORCE_CONSTANTS' or
        'force_constants.hdf5', a file contains force constants.
        Default is None.
        The priority for force constants is
        force_constants_filename > force_sets_filename > 'FORCE_SETS'.
    use_alm : bool, optional
        Default is False.
    factor : float, optional
        Phonon frequency unit conversion factor. Unless specified, default
        unit conversion factor for each calculator is used.
    frequency_scale_factor : float, optional
        Factor multiplied to calculated phonon frequency. Default is None,
        i.e., effectively 1.
    symprec : float, optional
        Tolerance used to find crystal symmetry. Default is 1e-5.
    is_symmetry : bool, optional
        Setting False, crystal symmetry except for lattice translation is not
        considered. Default is True.
    log_level : int, optional
        Verbosity control. Default is 0.

    """

    cell, smat, pmat = _get_cell_settings(unitcell_filename,
                                          supercell_filename,
                                          unitcell,
                                          supercell,
                                          calculator,
                                          primitive_matrix,
                                          supercell_matrix,
                                          symprec)

    # units keywords: factor, nac_factor, distance_to_A
    units = get_default_physical_units(calculator)
    if factor is None:
        _factor = units['factor']
    else:
        _factor = factor
    phonon = Phonopy(cell,
                     smat,
                     primitive_matrix=pmat,
                     factor=_factor,
                     frequency_scale_factor=frequency_scale_factor,
                     symprec=symprec,
                     is_symmetry=is_symmetry,
                     log_level=log_level)
    _set_nac_params(phonon,
                    nac_params,
                    born_filename,
                    is_nac,
                    units['nac_factor'])
    _set_force_constants(phonon,
                         force_constants_filename,
                         force_sets_filename,
                         use_alm)
    return phonon

def _get_cell_settings(unitcell_filename,
                       supercell_filename,
                       unitcell,
                       supercell,
                       calculator,
                       pmat,
                       smat,
                       symprec):
    if unitcell_filename is not None:
        cell, filename = read_crystal_structure(
            filename=unitcell_filename, interface_mode=calculator)
        _smat = _get_supercell_matrix(smat)
        _pmat = pmat
    elif supercell_filename is not None:
        cell, filename = read_crystal_structure(
            filename=supercell_filename, interface_mode=calculator)
        _smat = np.eye(3, dtype='intc', order='C')
        if pmat is None:
            _pmat = 'auto'
    elif unitcell is not None:
        cell = unitcell
    elif supercell is not None:
        cell = supercell
        _smat = np.eye(3, dtype='intc', order='C')
        if pmat is None:
            _pmat = 'auto'
    else:
        raise RuntimeError("Cell has to be specified.")

    if cell is None:
        msg = "'%s' could not be found." % filename
        raise FileNotFoundError(msg)

    _pmat = _get_primitive_matrix(_pmat, cell, symprec)

    return cell, _smat, _pmat

def _get_supercell_matrix(smat):
    if smat is None:
        _smat = np.eye(3, dtype='intc', order='C')
    elif len(np.ravel(smat)) == 3:
        _smat = np.diag(smat)
    elif len(np.ravel(smat)) == 9:
        _smat = np.reshape(smat, (3, 3))
    else:
        msg = "supercell_matrix shape has to be (3,) or (3, 3)"
        raise RuntimeError(msg)
    return _smat

def _get_primitive_matrix(pmat, unitcell, symprec):
    if type(pmat) is str and pmat in ('F', 'I', 'A', 'C', 'R', 'auto'):
        if pmat == 'auto':
            _pmat = guess_primitive_matrix(unitcell, symprec=symprec)
        else:
            _pmat = get_primitive_matrix_by_centring(pmat)
    elif pmat is None:
        _pmat = None
    elif len(np.ravel(pmat)) == 9:
        matrix = np.reshape(pmat, (3, 3))
        if matrix.dtype.kind in ('i', 'u', 'f'):
            det = np.linalg.det(matrix)
            if symprec < det and det < 1 + symprec:
                _pmat = matrix
            else:
                msg = ("Determinant of primitive_matrix has to be larger "
                       "than 0")
                raise RuntimeError(msg)
    else:
        msg = ("primitive_matrix has to be a 3x3 matrix, None, 'auto', "
               "'F', 'I', 'A', 'C', or 'R'")
        raise RuntimeError(msg)

    return _pmat

def _set_nac_params(phonon, nac_params, born_filename, is_nac, nac_factor):
    if nac_params is not None:
        _nac_params = nac_params
    elif born_filename is not None:
        _nac_params = parse_BORN(phonon.primitive, filename=born_filename)
    elif is_nac is True:
        if os.path.isfile("BORN"):
            _nac_params = parse_BORN(phonon.primitive, filename="BORN")
        else:
            raise RuntimeError("BORN file doesn't exist at current directory.")
    else:
        _nac_params = None

    if _nac_params is not None:
        if _nac_params['factor'] is None:
            _nac_params['factor'] = nac_factor
        phonon.set_nac_params(_nac_params)

def _set_force_constants(phonon,
                         force_constants_filename,
                         force_sets_filename,
                         use_alm):
    natom = phonon.supercell.get_number_of_atoms()

    if force_constants_filename is not None:
        dot_split = force_constants_filename.split('.')
        p2s_map = phonon.primitive.get_primitive_to_supercell_map()
        if len(dot_split) > 1 and dot_split[-1] == 'hdf5':
            fc = read_force_constants_hdf5(filename=force_constants_filename,
                                           p2s_map=p2s_map)
        else:
            fc = parse_FORCE_CONSTANTS(filename=force_constants_filename,
                                       p2s_map=p2s_map)
        phonon.set_force_constants(fc)
    elif force_sets_filename is not None:
        force_sets = parse_FORCE_SETS(natom=natom,
                                      filename=force_sets_filename)
        if force_sets:
            phonon.set_displacement_dataset(force_sets)
            phonon.produce_force_constants(
                calculate_full_force_constants=False,
                use_alm=use_alm)
    elif os.path.isfile("FORCE_SETS"):
        force_sets = parse_FORCE_SETS(natom=natom)
        if force_sets:
            phonon.set_displacement_dataset(force_sets)
            phonon.produce_force_constants(
                calculate_full_force_constants=False,
                use_alm=use_alm)


#coding:utf-8

import requests
import re
from bs4 import BeautifulSoup
from urllib import quote
from time import sleep
import threading
from time import sleep
#from urllib3.exceptions import NewConnectionError
TIME_SPLIT2=1

class User():
    def __init__(self,name):
        self.TIME_SPLIT=0.2
        self.name=name
        self.base_url = "http://300report.jumpw.com/list.html?name=%s"%name
        self.match_urls=[]
        self.match_info=[]    #保存每场比赛的信息,每个元素都是一个字典
        self.USER_VALID=False      #标记用户名是否存在

        html=requests.get(self.base_url)
        #print html.text
        soup=BeautifulSoup(html.text,'lxml')
        #print soup.tr
        tr=soup.find_all("tr")
        if len(tr)!=0:
            self.USER_VALID=True
            self.set_user_info(tr)

    def set_user_info(self,tr):
        self.user_info={ 'username':tr[0].td.next_sibling.string,\
                         'userlevel':int(tr[1].td.next_sibling.string),\
                         'jiecao':int(tr[2].td.next_sibling.string),
                         'totalwins':int(tr[3].td.next_sibling.string),\
                         'totalplays':int(tr[4].td.next_sibling.string)}

        #获取每个比赛详情页面的地址
        self.match_urls=self.find_match_url(self.base_url)

        #循环访问比赛详情页面,获取每局比赛的信息
        threads=[threading.Thread(target=self.find_match_content,args=(match_urlsi,)) \
                 for match_urlsi in self.match_urls]
        for t in threads:
            try:
                sleep(self.TIME_SPLIT)
                t.start()
            except:
                sleep(1)
                pass
        #match_urls = re.findall(re_match_url, html.text)

    def find_match_url(self,url):
        re_match_url=r"match\.html\?id\=\d+"
        re_match_url=re.compile(re_match_url)
        match_urls=[]
        pages=int((self.user_info['totalplays']-1)/10)  #每页10条信息,计算一共有多少页
        for i in range(1,pages+2):
            try:
                sleep(TIME_SPLIT2)
                content=requests.get(self.base_url+"&index=%s"%str((i-1)*10)).text
                tmplist=re.findall(re_match_url,content)        #抓取比赛详情页面的url
                if len(tmplist)!=0:
                    for l in tmplist:
                        match_urls.append("http://300report.jumpw.com/"+l)
            except:
                sleep(TIME_SPLIT2)
                continue
        return match_urls

    #获取每场比赛的详细数据
    def find_match_content(self,match_url):
        content=requests.get(match_url).text
        match_id=int( match_url.split('=')[1] )
        soup=BeautifulSoup(content,'lxml')
        tr=soup.find_all('tr')
        for t in tr:
            if re.search(self.name,t.text):
                tds=[td for td in t.children]
                this_match_info={"id":match_id,\
                                 "hero":tds[3].text.split(")")[1].split("(")[0],\
                                 'kills':tds[5].string,\
                                 'wins':True if (tds[7].parent.parent.previous_element.previous_element==u"胜利") else False,\
                                 'buildings':int(tds[9].string),\
                                 'soldiers':int(tds[11].string),\
                                 'golds':int(tds[13].string),\
                                 'score':tds[15].string,\
                                 'exps':tds[21].string,\
                                 'jiecao':int(tds[23].string),\
                                 'wins_p':tds[25].string,}
        self.match_info.append(this_match_info)
        return this_match_info


# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from setuptools import setup

install_requires = [
    "flask==0.10.1",
    "requests==2.0.0",
    "mongoengine==0.8.4",
    "flask_mongoengine==0.7.0",
]

setup(name="laughing-happiness",
    version="0.0",
    description="",
    install_requires=install_requires,
)


#!venv/bin/python
from flask import Flask, request, render_template, jsonify
from strava_monitoring.fetcher import get_distances

app = Flask(
    __name__, template_folder='strava_monitoring\\templates')

@app.route('/')
@app.route('/index')
def index():
    distances = get_distances()
    return render_template('index.html', distances=distances)

if __name__ == '__main__':
    app.run(debug=True, host='0.0.0.0')


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('my_blog', '0001_initial'),
    ]

    operations = [
        migrations.AlterModelOptions(
            name='user',
            options={'ordering': ['-id'], 'verbose_name': '\u7528\u6237', 'verbose_name_plural': '\u7528\u6237'},
        ),
        migrations.RemoveField(
            model_name='comment',
            name='user',
        ),
        migrations.AddField(
            model_name='comment',
            name='artical',
            field=models.ForeignKey(verbose_name=b'\xe6\x96\x87\xe7\xab\xa0', blank=True, to='my_blog.Artical', null=True),
        ),
        migrations.AlterField(
            model_name='artical',
            name='tag',
            field=models.ManyToManyField(to='my_blog.Tag', verbose_name='\u6807\u7b7e'),
        ),
    ]


import time
from selenium import webdriver

# 使用chrome的驱动器
browser = webdriver.Chrome()
# 设置超时阈值（最好设置长一点）
browser.set_page_load_timeout(120)
# 打开页面，由于页面动态加载，需要模拟滚动页面，让其加载完成
browser.get('http://www.acfun.cn/search/?#query=南方公园')
browser.execute_script("window.scrollTo(0, document.body.scrollHeight);")
# 根据目测知道有多少页，右键检查，复制为selector
page_info = browser.find_element_by_css_selector('#list-pager > div > span.hint')
# 页号信息文本格式为：当前页: 1/39页
# 先用/分开取第二个，并去除最后一个字
pages = int(page_info.text.split('/')[1][:-1])
print('总页数: %d' % (pages))
# 接着去遍历页数
for page in range(1, pages + 1):
    if page > 3:
        break
    print('当前页面: %d' % (page))
    # 遍历页面
    url = 'http://www.acfun.cn/search/?#page=' + str(page) + ';query=南方公园;type=video'
    browser.get(url)
    browser.execute_script("window.scrollTo(0, document.body.scrollHeight);")
    time.sleep(3)   # 等待滚动的时间
    # 逐层取selector，第一层找到视频列表（层级关系可以简化)
    # 层级关系为list-wrap -> video-cell clearfix -> div ->[div title, video info, description, video-tags-and-channel](更深略)
    vlist = browser.find_element_by_css_selector('#main > section > div.column-left > div.video-list.list-warp.active > div.video-list-wrap')
    # 列表下取所有video-cell clearfix标签，注意element和elements，这里使用 xpath 形式取标签
    velems = vlist.find_elements_by_xpath("//*[@class='video-cell clearfix']")
    for velem in velems:
        # 往下有一层div
        vitem = velem.find_element_by_tag_name('div')
        # 取得播放地址，title和播放信息
        vtitle = vitem.find_element_by_css_selector('div.title').find_element_by_tag_name('a')
        # 取得链接
        vlink = vtitle.get_attribute('href')
        # 取得信息
        vinfo = vitem.find_element_by_css_selector('div.video-info.clearfix').find_element_by_tag_name('div')
        print(vtitle.text, vinfo.text, vlink)


import web

from web import form
from globals import *
from lesson import *
from utils import *

class display:
    def GET(self):
        data = web.input()
        #session = web.config.get('_session')
        session = web.config._session
        if 'page' in data:
            lesson_data = readLessonData(convertToRealPath(data['page']))
            storeVisit(lesson_data['index'], session.user)

            if lesson_data['type'] == 'lesson':
                raise web.seeother('/lesson?page=%s' % (data['page']))
            elif lesson_data['type'] == 'quiz':
                raise web.seeother('/quiz?page=%s' % (data['page']))

        return render.lesson(getLesson(session.user, ''))

    def POST(self):
        return self.GET()


from pandas import json

import sklearn
import json

#
# We implement cosine similarity here.
#
def calculateSim(path, mat, user_id):
	with open('%s_user_id.json' % path, 'r') as f:
		train_user_id = json.load(f)

	num = train_user_id[user_id]

	temDict = sklearn.metrics.pairwise.cosine_similarity(mat[num].reshape(1, -1), mat)
	result = {}
	i = 0
	for key,value in train_user_id.items():
		result[key] = temDict[0][i]
		i+=1
	# result = sorted(result.items(), key=lambda d: d[1], reverse=True)
	print("Similarity calculation done! Start writing.\n")

	# with open("%s_sim_result.json" % path,"w") as fp:
	# 	json.dump(result,fp, sort_keys=False)

	with open('%s_simi_result.json' % path,'w') as fp:
		json.dump(result,fp)


import numpy as np
import pyaudio
from scipy.fftpack import fft, fftfreq
import scipy
import struct
from random import randint

class g_soundrandom():
    def __init__(self):
        self.amount = 1
        # sound2light stuff
        self.sample_rate = 44100
        self.buffer_size = 2**11
        self.thres_factor = 0
        self.channel = 1

        pa = pyaudio.PyAudio()
        chosen_device_index = -1
        for x in range(0,pa.get_device_count()):
            info = pa.get_device_info_by_index(x)
            if info["name"] == "pulse":
                chosen_device_index = info["index"]

        p = pyaudio.PyAudio()

        self.stream = p.open(
            format = pyaudio.paInt16,	# paFloat32 vs paInt16 ?
            channels = 1,
            rate = self.sample_rate,
            input = True,
            output = False,
            frames_per_buffer = self.buffer_size
            )

        self.threshold = 0.5

    def control(self, amount, threshold, channel):
        self.amount = amount*6
        self.threshold = threshold
        self.channel = int(channel*4)

#        self.thres_list = self.thres_list*self.thres_factor
        # new_list = [x+1 for x in my_list]
#        self.thres_list = [x+self.thres_factor for x in self.thres_list]

    def label(self):
        return ['amount',round(self.amount,2),
                'threshold',round(self.threshold,2),
                'channel',round(self.channel)]

    def generate(self, step, world):
        tempworld = np.zeros([10, 10, 10])

        number = int(self.amount*(self.update_line()[self.channel]-self.threshold))

        for i in range(number):
            world[:, randint(0, 9), randint(0, 9), randint(0, 9)] = 1.0

        return np.clip(world, 0, 1)

    def get_fft(self, data):
        FFT = fft(data)                                # Returns an array of complex numbers
        freqs = fftfreq(self.buffer_size, 1.0/self.sample_rate)  # Returns an array containing the frequency values

        y = abs(FFT[0:int(len(FFT)/2)])/1000                # Get amplitude and scale
        y = scipy.log(y) - 2                           # Subtract noise floor (empirically determined)
        return (freqs, y)

    def control_threshold(self, dat, thres):
        if dat < thres:
            return 0.0
        else:
            return dat

    def update_line(self):
        buf = self.stream.read(self.buffer_size)
        data = scipy.array(struct.unpack("%dh"%(self.buffer_size), buf))

        freqs, y = self.get_fft(data)

        # Normalize
        y = y / 5.0

        # Average into chunks of N
        N = 100

        yy = [scipy.average(y[n:n+N]) for n in range(0, len(y), N)]

        yy = yy[:int(len(yy)/2)] # Discard half of the samples, as they are mirrored

        # now do some threshold detection
        # for i in range(len(yy)):
        #    yy[i] = self.control_threshold(yy[i], self.threshold)

        return np.round(np.clip(yy,0,1),2)


from untwisted.network import core, xmap, cmap, READ, WRITE, spawn, zmap
from untwisted.utils.stdio import DUMPED, CLOSE, LOAD
from collections import deque

class Stdin(object):
    """
    A protocol for sending data through a device.
    """

    def __init__(self, device):
        self.device = device
        device.dump = self.dump
        self.queue  = deque()

    def dump(self, data):
        """
        It sends data asynchronously through the device.
        """

        self.queue.append(data)
        xmap(self.device, WRITE, self.update)

    def update(self, device):
        try:
            data = self.queue.popleft()
        # As device.write returns None
        # we don't need to worry whether it has sent all the bytes
        # unlike with sockets.
            device.write(data)
        except IndexError:
        # When the queue is empty then we don't need
        # the WRITE event.
            zmap(device, WRITE, self.update)
            spawn(device, DUMPED)
        except IOError as excpt:
        # If something went wrong it spawns CLOSE with err.
        # The err parameter gives a clue of what happened.
            err = excpt.args[0]
            spawn(device, CLOSE, err)

class Stdout(object):
    """
    A protocol for reading data from a device.
    """

    def __init__(self, device):
        xmap(device, READ, self.update)

    def update(self, device):
        try:
            data = device.read()
            spawn(device, LOAD, data)
        except IOError as excpt:
        # If something went wrong it spawns CLOSE with err.
            err = excpt.args[0]
            spawn(device, CLOSE, err)

def lose(device):
    """
    It is used to close the device and destroy the Device instance.
    """
    device.destroy()
    device.close()


import os
import json
import pickle

data_path = os.path.join(os.getcwd(), "..", "data")
train_one_week_path = os.path.join(data_path, "train_one_week")
test_one_week_path = os.path.join(data_path, "test_one_week")

user_ids_train_path = os.path.join(data_path, "user_ids_train")
user_ids_test_path = os.path.join(data_path, "user_ids_test")

all_items_path = os.path.join(data_path, "contentdata")
all_articles = os.listdir(all_items_path)

"""
print("UNSEEN ARTICLES")
i = 0
j = 0
line_no = 0
unseen_articles = set()
user_with_articles = set()
with open(train_one_week_path, "r") as infile:
    while True:
        line = infile.readline()
        if line == "":
            break
        else:
            json_object = json.loads(line)
            line_no += 1
            print(line_no)
            try:
                item_id = json_object["id"]
                active_time = json_object["activeTime"]
                i += 1
                user_with_articles.add(json_object["userId"])
                if item_id not in all_articles:
                    j += 1
                    unseen_articles.add(item_id)
            except KeyError:
                continue
print(i)
print(j)
print(unseen_articles)
print("NUMBER OF USERS WITH ARTICLES")
print(len(user_with_articles))

"""

full_data_path = os.path.join(data_path, "contentdata")
articles = os.listdir(full_data_path)
item_ids_path = os.path.join(data_path, "clean", "item_ids.json")
with open(item_ids_path, "r") as infile:
    ids = json.load(infile)

print("NUMBER OF POSSIBLE ARTICLES")
print(len(articles))
print("NUMBER OF USABLE ARTICLES")
print(len(ids))

i = 0
j = 0
line_no = 0
with open(train_one_week_path, "r") as infile:
    while True:
        line = infile.readline()
        if line == "":
            break
        else:
            json_object = json.loads(line)
            line_no += 1
            print(line_no)
            try:
                item_id = json_object["id"]
                active_time = json_object["activeTime"]
                if item_id in all_articles:
                    i += 1
                    if item_id in ids:
                        j += 1
            except KeyError:
                continue

print("NUMBER OF ARTICLES SEEN")
print(i)
print("NUMBER OF USABLE ARTICLES SEEN")
print(j)

"""
print("TRAIN DATASET")
i = 0
user_ids_train = set()
with open(train_one_week_path, "r") as infile:
    while True:
        line = infile.readline()
        if line == "":
            break
        else:
            i += 1
            print("line" + str(i) + " - NB IDS: " + str(len(user_ids_train)))
            json_object = json.loads(line)
            id = json_object["userId"]
            user_ids_train.add(id)
print(i)
print("Export IDS")
with open(user_ids_train_path, "wb") as outfile:
    pickle.dump(user_ids_train, outfile)

print("TEST DATASET")
i = 0
user_ids_test = set()
with open(test_one_week_path, "r") as infile:
    while True:
        line = infile.readline()
        if line == "":
            break
        else:
            i += 1
            print("line" + str(i) + " - NB IDS: " + str(len(user_ids_test)))
            json_object = json.loads(line)
            id = json_object["userId"]
            user_ids_test.add(id)
print(i)
print("Export IDS")
with open(user_ids_test_path, "wb") as outfile:
    pickle.dump(user_ids_test, outfile)

with open(user_ids_train_path, "rb") as infile:
    user_ids_train = pickle.load(infile)
with open(user_ids_test_path, "rb") as infile:
    user_ids_test = pickle.load(infile)

i = 0
for test_id in list(user_ids_test):
    if test_id not in user_ids_train:
        i += 1
print("NB test users not in train: " + str(i))

# TO MAKE SURE THE INFO IN TEST IS NOT PRESENT INTO TRAIN
# cat train_one_week | grep cx:3756qzoqoc8wy2rk0bei3l4v >
# train_cx:3756qzoqoc8wy2rk0bei3l4v.txt
# cat test_one_week | grep cx:3756qzoqoc8wy2rk0bei3l4v >
# test_cx:3756qzoqoc8wy2rk0bei3l4v.txt
# then execute code below
train_user_info_path = os.path.join(data_path, "test_cx:3756qzoqoc8wy2rk0bei3l4v.txt")
test_user_info_path = os.path.join(data_path, "train_cx:3756qzoqoc8wy2rk0bei3l4v.txt")
train_user_info = list()
with open(train_user_info_path, "r") as infile:
    for line in infile.readlines():
        train_user_info.append(json.loads(line))
test_user_info = list()
with open(test_user_info_path, "r") as infile:
    for line in infile.readlines():
        test_user_info.append(json.loads(line))
i = 0
for record in test_user_info:
    if record in train_user_info:
        print(record)
        i += 1
print("number of duplicated records: " + str(i))
"""


import time
import requests
from lxml import etree
import dbmodle

def getquanwang():
    url = 'http://www.goubanjia.com/'
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36'
    }
    r = requests.get('http://127.0.0.1:5000/http', timeout=30)
    proxies = r.json()
    print(proxies)
    r = requests.get(url, headers=headers, timeout=30)
    return r

def analyzequanwang(r):
    tree = etree.HTML(r.text)
    tr_list = tree.xpath('//table/tbody/tr')
    print(tr_list)
    _ = []
    for tr in tr_list:
        host_list = tr.xpath('./td[1]/*[not(contains(@style,"none"))]/text()')[:-1]
        host = "".join(host_list)
        # print(tr.xpath('./td[1]/*[last()]/@class'),tr.xpath('./td[1]/*[last()]/text()'))
        portnum = tr.xpath('./td[1]/*[last()]/@class')[0].split(' ')[1]
        # print(portnum)
        portlist = [str('ABCDEFGHIZ'.index(port)) for port in portnum]
        # print(_)
        result = int("".join(portlist))
        port = result >> 3
        protocol = tr.xpath('./td[3]/a/text()')[0]
        print(protocol, host, port)
        _.append({
            protocol: f'{protocol}://{host}:{port}'
        })
    return _
if __name__ == '__main__':
    while 1:
        try:
            r = getquanwang()
            result = analyzequanwang(r)
            print('result', result)
            for proxy in result:
                for protocol, url in proxy.items():
                    print("save ", proxy)
                    if 'https' in url:
                        dbmodle.sethttps(url)
                    else:
                        dbmodle.sethttp(url)
        except Exception as e:
            print(e)
            # raise
        time.sleep(30)


import os
import configparser
import requests
import sqlalchemy as sa
from requests_cache import CachedSession
from requests_futures.sessions import FuturesSession
from concurrent.futures import as_completed

from tqdm import tqdm
import sys

def getcorps(corplist):
    corpfuture = []
    print("get corps")
    for corpid in corplist:
        if isinstance(corpid, str) and corpid.startswith("https"):
            corpfuture.append(session.get(str(corpid)))
        else:
            corpfuture.append(session.get(corplookupurl.format(corpid)))
    badlist = []
    pbar = tqdm(total=len(corplist))
    for corpdata in as_completed(corpfuture):
        if corpdata.result().status_code == 200:
            corpid = int(str(corpdata.result().url).split('/')[5])
            corpjson = corpdata.result().json()
            connection.execute(invNames.insert(),
                               itemID=corpid,
                               itemName=corpjson.get('name', None)
                               )

            connection.execute(crpNPCCorporations.insert(),
                               corporationID=corpid,
                               description=corpjson.get('description', None),
                               )
        else:
            badlist.append(corpdata.result().url)
            print(corpdata.result().url)
        pbar.update(1)
    return badlist

if len(sys.argv) < 2:
    print("Load.py destination")
    exit()

database = sys.argv[1]

if len(sys.argv) == 3:
    language = sys.argv[2]
else:
    language = 'en'

fileLocation = os.path.dirname(os.path.realpath(__file__))
inifile = fileLocation + '/sdeloader.cfg'
config = configparser.ConfigParser()
config.read(inifile)
destination = config.get('Database', database)
sourcePath = config.get('Files', 'sourcePath')
redis_server = config.get('Redis', 'server')
redis_db = config.get('Redis', 'db')

schema = None
if database == "postgresschema":
    schema = "evesde"

engine = sa.create_engine(destination, echo=False)
metadata = sa.MetaData(schema=schema)
connection = engine.connect()
trans = connection.begin()

crpNPCCorporations = sa.Table('crpNPCCorporations', metadata,
                              sa.Column('corporationID', sa.INTEGER(), primary_key=True, autoincrement=False, nullable=False),
                              sa.Column('size', sa.CHAR(length=1)),
                              sa.Column('extent', sa.CHAR(length=1)),
                              sa.Column('solarSystemID', sa.INTEGER()),
                              sa.Column('investorID1', sa.INTEGER()),
                              sa.Column('investorShares1', sa.INTEGER()),
                              sa.Column('investorID2', sa.INTEGER()),
                              sa.Column('investorShares2', sa.INTEGER()),
                              sa.Column('investorID3', sa.INTEGER()),
                              sa.Column('investorShares3', sa.INTEGER()),
                              sa.Column('investorID4', sa.INTEGER()),
                              sa.Column('investorShares4', sa.INTEGER()),
                              sa.Column('friendID', sa.INTEGER()),
                              sa.Column('enemyID', sa.INTEGER()),
                              sa.Column('publicShares', sa.INTEGER()),
                              sa.Column('initialPrice', sa.INTEGER()),
                              sa.Column('minSecurity', sa.FLOAT()),
                              sa.Column('scattered', sa.Boolean(name='cnpcc_scatt')),
                              sa.Column('fringe', sa.INTEGER()),
                              sa.Column('corridor', sa.INTEGER()),
                              sa.Column('hub', sa.INTEGER()),
                              sa.Column('border', sa.INTEGER()),
                              sa.Column('factionID', sa.INTEGER()),
                              sa.Column('sizeFactor', sa.FLOAT()),
                              sa.Column('stationCount', sa.INTEGER()),
                              sa.Column('stationSystemCount', sa.INTEGER()),
                              sa.Column('description', sa.VARCHAR(length=4000)),
                              sa.Column('iconID', sa.INTEGER()),
                              schema=schema
                              )

invNames = sa.Table('invNames', metadata,
                    sa.Column('itemID', sa.INTEGER(), primary_key=True, autoincrement=False, nullable=False),
                    sa.Column('itemName', sa.VARCHAR(length=200), nullable=False),
                    schema=schema
                    )

maincorplist = []

corpurl = "https://esi.evetech.net/latest/corporations/npccorps/?datasource=tranquility"

corplookupurl = 'https://esi.evetech.net/latest/corporations/{}/?datasource=tranquility'

errorcount = 0

base_session = CachedSession("evesde", backend="redis")

reqs_num_workers = 50

session = FuturesSession(max_workers=reqs_num_workers, session=base_session)

corps = requests.get(corpurl).json()

firstbadlist = getcorps(corps)
print("Getting badlist")
secondbadlist = getcorps(firstbadlist)

trans.commit()