# AUTHOR: JT MAHONEY
# DATE: 08FEB2019

import numpy as np
from helper_fxns.pixelshift37 import *
from helper_fxns.roi import ROIEncoder, ROIRect, ROIDecoder
import tifffile
from skimage.feature import peak_local_max
from skimage.draw import polygon_perimeter
from skimage import io
from sklearn.cluster import DBSCAN
from bokeh.plotting import figure, show
from bokeh.io import output_notebook
from bokeh.palettes import Category20 as palette
import itertools
import collections
import pandas as pd

import mahotas as mh
from skimage.filters import threshold_otsu
from skimage.transform import downscale_local_mean

import os
import shutil
import glob
import zipfile

import json
from PIL import Image

from unet.mahoney import split_2d_img, all_the_kings_men
from keras.preprocessing.image import img_to_array, load_img
from unet import utils

def generate_json(channel_dictionary, dichroic_dictionary, output_directory):
# INPUT:
#   ch_dictionary:
#   file_name:
#   ch_colors:
    save_path = os.path.join(output_directory, 'image_data.json')

    img_data = {
        "channel_dictionary":channel_dictionary,
        "dichroic_dictionary":dichroic_dictionary
    }

    with open(save_path, 'w+') as write_file:
        json.dump(img_data, write_file)

def read_json(data_file):
    with open(data_file, "r") as read_file:
        data = json.load(read_file)
        return(data)

def shrink_image(multi_ch_img, ch_dict, shrink_scale):
    image_stack = []
    for key, value in ch_dict.items():
        img = Image.fromarray(multi_ch_img[key,:,:])
        h = int(multi_ch_img[key,:,:].shape[0]/shrink_scale)
        w = int(multi_ch_img[key,:,:].shape[1]/shrink_scale)
        img = img.resize((w,h), Image.ANTIALIAS)
        np_img = np.array(img)
        image_stack.append(np_img)
    shrunk_img = np.dstack(image_stack)
    shrunk_img = np.moveaxis(shrunk_img, -1, 0)
    return(shrunk_img)

def shrink_roi_labels(dimension_list,shrink_scale):
    loader_list = []
    for n in dimension_list:
        nn = int(n/shrink_scale)
        loader_list.append(nn)
    return(loader_list)

def save_png(multi_ch_img, ch_dict, destination_folder):
    for key, value in ch_dict.items():
        img = Image.fromarray(multi_ch_img[key])
        save_file = value + '_max-z.png'
        save_file = os.path.join(destination_folder, save_file)
        img.save(save_file)

def save_png_with_rois(multi_ch_img, ch_dict, destination_folder, df_all_rois):
    for key, value in ch_dict.items():
        img = multi_ch_img[key]
        save_file = value + '_labeled.png'
        save_file = os.path.join(destination_folder, save_file)

        single_channel_df = df_all_rois.loc[df_all_rois['channel_name']==value]
        for index, col in single_channel_df.iterrows():
            x1 = col['x1']
            y1 = col['y1']
            x2 = col['x2']
            y2 = col['y2']
            x_coords = [x1, x1, x2, x2]
            y_coords = [y1, y2, y2, y1]

            rr, cc = polygon_perimeter(y_coords, x_coords, shape=img.shape)
            try:

                img[rr,cc]=np.max(img)
            except:
                print("ERROR DRAWING RECTANGLE AT: ", rr, cc)
        io.imsave(save_file, img)

def convert_16_to_8(np_img):
# INPUT:
#     np_img: numpy image.  shape = 2D or 3D

# OUTPUT:
#     return: 8 bit version of origional np_img

    info = np.iinfo(np_img.dtype)
    if np.issubdtype(np_img.dtype, np.dtype('uint16')):
        data = np_img.astype(np.int16)/4095
        # print('ORIGIONAL IMG: ', np.max(np_img))
        # print('CONVERSION: ', np.max(data), " INFO: ", info)
        data = 255 * data
        img8 = data.astype(np.uint8)
        return(img8)
    elif np.issubdtype(np_img.dtype, np.dtype('uint8')):
        return(np_img)

def pixel_shift_3d(np_img, dichroic_dictionary, json_file):
# INPUT:
#     np_img: numpy image. shape = 3D
#     dichroic_dictionary: dictionary where key=channel number, value=name of dichroic_dictionary
#     json_file: path to the .json file containing the shift values for each dichroic generated by the shift callibration script

# OUTPUT:
#     return: numpy image of shape (n_channels, height, width).  height and width may be different than origional image

    number_of_channels = np_img.shape[0]
    shifted_list = []

    # instantiate PixelShifter class object from pixelshift37
    shift_obj = PixelShifter(jsonfilepath=json_file)

    # perform shift on each channel
    for ch in range(np_img.shape[0]):
        ch_img = np_img[ch,:,:]
        ch_dichroic = dichroic_dictionary[ch]
        for key in shift_obj.shiftdict.keys():
            if key.split('_')[-1].lower() in ch_dichroic.lower():
                ch_img = shift_obj.shape_images(ch_img)
                shiftval = shift_obj.shiftdict[key]
        img = shift_obj.align_with_shift_matrix(ch_img,shiftval)
        shifted_list.append(img)

    # create numpy array where .shape = 3 from list of lists 'shifted_list'
    shifted_img = np.dstack(shifted_list)

    # rearranges shape of numpy array to look more like origional image
    shifted_img = np.moveaxis(shifted_img, -1, 0)
    print("shifted img shape: ", shifted_img.shape)

    return(shifted_img)

def correct_shift_upsampling(img):
    if np.issubdtype(img.dtype, np.dtype('uint16')):
        over_values = img > 4095
        img[over_values] = 4095
        return(img)
    elif np.issubdtype(img.dtype, np.dtype('uint8')):
        over_values = img > 255
        img[over_values] = 255
        return(img)

def ml_model_segment(single_channel_array, model, orig_x_dim=2048, orig_y_dim=2048, diff_out_size=True, mask_threshold=0.5, mask_min_pixel=300):
    img = single_channel_array
    img = img_to_array(img)
    img /= 255

    pv_model = model

    img_slices = split_2d_img(img, x_dim=orig_x_dim, y_dim=orig_x_dim, diff_out_size=diff_out_size) # gets converted from np.shape = 3 to np.shape = 4

    predicted_img_slices_list = []
    for s in img_slices:
        pred_new = pv_model.predict(np.asarray([s['slice_img']]))
        pred_new_list = [pred_new[i] for i in range(pred_new.shape[0])]
        # pred_new_mask = utils.label_mask(pred_new_list[0])
        s['mask'] = pred_new_list[0]
        predicted_img_slices_list.append(s)

    p_img, p_mask = all_the_kings_men(predicted_img_slices_list, img.shape[1], img.shape[0])
    lab_mask = utils.label_mask(p_mask, threshold=mask_threshold, min_pixel=mask_min_pixel)

    print('LAB MASK SHAPE: ', lab_mask.shape)

    region_props=regionprops(lab_mask,intensity_image=p_img)
    model_bb_array = bbs_from_rprops(region_props)
    return(model_bb_array, lab_mask)

def dot_segment(single_channel_array, threshold=40, max_peaks=6000, min_samples_for_cluster=15):
# INPUT:
#     single_channel_array: numpy image of single channel
#     threshold: minimum value of threshold for subtracting background.  anything less than this value is given a value of 0
#     max_peaks: maximum number of dots to be counted in a single channel image
#     min_samples_for_cluster: minimum number of peaks required to count as a cell when clustered with DBSCAN
# OUTPUT:
#     bb_array: a numpy array for the coordinates of each bounding box
#     dot_coords: coordinates of each peak detected (mostly for troubleshooting.  value usually dropped ,_)

    dot_img = single_channel_array

    # get coordinates of peaks/dots
    dot_coords = peak_local_max(dot_img, min_distance=2, threshold_abs=threshold, num_peaks=max_peaks)
    X = np.column_stack([dot_coords[:,0], dot_coords[:,1]])

    # generate clusters from regions of dots with adiquate densities
    db = DBSCAN(eps=25., min_samples=min_samples_for_cluster).fit(dot_coords)

    # generate boolean masks and labels from db clusters
    core_samples_mask = np.zeros_like(db.labels_, dtype=bool)
    core_samples_mask[db.core_sample_indices_] = True
    labels = db.labels_

    n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0)
    counter = collections.Counter(labels)

    # create array and fill with coordinates of cluster bounding boxes
    # colors not used at the moment.  used to be used to generate a cluster graph where each cluseter was a different color
    labels_reshape = labels.reshape(labels.shape[0], 1)
    labeled_coords = np.hstack((dot_coords, labels_reshape))
    unique_labels = set(labels)
    colors = itertools.cycle(palette[10])
    bb_array = np.empty((0,5), dtype=int)

    for k, col in zip(unique_labels, colors):
        if k == -1:
            col = [0,0,0,1]

        class_member_mask = (labels == k)
        xy = X[class_member_mask & core_samples_mask]
        if xy.shape[0] != 0:
            x_min = np.amin(xy[:,1])
            x_max = np.amax(xy[:,1])
            y_min = np.amin(xy[:,0])
            y_max = np.amax(xy[:,0])

            k_list = [k, x_min, x_max, y_min, y_max]
            bb_array = np.append(bb_array, [k_list], axis=0)

    return(bb_array, dot_coords)

def watershed_single_channel(np_array, threshold=10, blur_factor=9, max_bb_size=13000, min_bb_size=1000, footprint=10):
# INPUT:
#     np_array: numpy image of single channel
#     threshold: minimum value to be analysed
#     blur_factor: gaussian blur.  some blur is good.  too much is bad.  too little is bad too.
#     max_bb_size: set maximum size for a bounding box.
#     min_bb_size: set minimum size for a bounding box.
#     footprint: box of size footprint x footprint used to find regions of maximum intensity.
# OUTPUT:
#     return: array of bounding boxes for the channel image given
# NOTE: input values should be tuned.  this alg is useless if input arguments are not optimized... right now its by hand.  otherwise just leave them

    # nuclear is a blured version of origional image
    nuclear = mh.gaussian_filter(np_array, blur_factor)
    # calculate a minimum threshold using otsu method
    otsu_thresh = threshold_otsu(nuclear)

    # determine minimum threshold from otsu and input argument.  if little/no signal in image; otsu value can be way too low
    set_thresh = None
    if threshold>otsu_thresh:
        set_thresh = threshold
    else:
        set_thresh = otsu_thresh

    # set values lower than set_thresh to zero
    index_otsu = nuclear < set_thresh
    nuclear[index_otsu] = 0

    # determine areas of maximum intensity and the distance between them
    thresh = (nuclear > nuclear.mean())
    dist = mh.stretch(mh.distance(thresh))
    Bc = np.ones((footprint, footprint))

    # the code the generate region_props from watersheding alg.
    maxima = mh.morph.regmax(dist, Bc=Bc)
    spots, n_spots = mh.label(maxima, Bc=Bc)
    sizes = mh.labeled.labeled_size(spots)
    too_big = np.where(sizes > max_bb_size)
    spots = mh.labeled.remove_regions(spots, too_big)
    spots = mh.labeled.remove_bordering(spots)
    spots, n_left = mh.labeled.relabel(spots)
    surface = (dist.max() - dist)
    areas = mh.cwatershed(surface, spots)
    areas *= thresh

    # get list of region properties from watershed.  allot of information in region_props.  allot of which is inaccurate.  NEVER TRUST REGIONPROPS!
    region_props=regionprops(areas,intensity_image=nuclear)

    # generate array of bounding boxes from measured region properties. call bbs_from_rprops()
    watershed_bb_array = bbs_from_rprops(region_props, max_bb_size, min_bb_size)

    return(watershed_bb_array)

def bbs_from_rprops(rprop_list, max_bb_size=10000, min_bb_size=1):
# INPUT:
#     rprop_list: origional region properties list. has allot of unecessary information
#     max_bb_size: maximum bounding box size allowed
#     min_bb_size: minimum bounding box size allowed
# OUTPUT:
#     return: array of bouding boxes generated with the watershedding alg.

    watershed_bb_list = []
    for cell_n in range(len(rprop_list)):
        rprops = rprop_list[cell_n]
        y1 = rprops['bbox'][0]
        x1 = rprops['bbox'][1]
        y2 = rprops['bbox'][2]
        x2 = rprops['bbox'][3]
        area = (abs(y2-y1)*abs(x2-x1))

        # gate to make sure bbs are the correct size
        if area > min_bb_size and area < max_bb_size:
            watershed_bb_list.append([cell_n,x1,x2,y1,y2])

    water_bbs = np.asarray(watershed_bb_list)
    return(water_bbs)

def make_roi_files(segmented_cells_csv, output_dir):
    df = pd.read_csv(segmented_cells_csv)
    unique_ch_names = df.channel_name.unique()

    for ch in unique_ch_names:
        ch_df = df[df.channel_name == ch]
        d_path = output_dir + '/' + ch + '_RoiSet/'
        zip_url = output_dir + '/' + ch + '_RoiSet'

        if not os.path.exists(d_path):
            os.makedirs(d_path)

        for inx, row in ch_df.iterrows():
            x1 = row['x1']
            y1 = row['y1']
            x2 = row['x2']
            y2 = row['y2']

            f_string = d_path + str(inx) + '.roi'
            roi_obj = ROIRect(y1,x1,y2,x2)
            with ROIEncoder(f_string, roi_obj) as roi:
                roi.write()

        shutil.make_archive(zip_url, 'zip', d_path)
        shutil.rmtree(d_path)

# def read_imagej_roi_zip(filename):

#     roi_list = []
#     with zipfile.ZipFile(filename) as zf:
#         for name in zf.namelist():
#             roi_path = zf.extract(name, '/tmp')
#             roi = read_roi(roi_path)
#             if roi is None:
#                 continue

#             # label = str(name).rstrip('.roi')
#             # if dict_format:
#             #     roi_list.append({'label': label, 'polygons': roi.T})
#             # else:
#             #     roi_list.append([label, roi])
#             roi_list.append(roi)
#         return roi_list

# def read_roi(roi_path):
#     try:
#         with ROIDecoder(roi_path) as roi:
#             r = roi.get_roi()
#             #print(r.name)
#             roi_dictionary = {}
#             roi_dictionary['top'] = r.header['TOP']
#             roi_dictionary['bottom'] = r.header['BOTTOM']
#             roi_dictionary['left'] = r.header['LEFT']
#             roi_dictionary['right'] = r.header['RIGHT']
#             # r.x_coords = r.left + r.x_coords
#             # r.y_coords = r.top + r.y_coords
#             return(roi_dictionary)

#     except Exception as other:
#             print(roi_path, other)
#             return None

def read_imagej_roi_zip(filename):
    from roi_fxns import ijroi
    rois = ijroi.read_roi_zip(filename)
    roi_list = []
    for r in rois:
        # print(r)
        r = r[1]
        ys = r[:,0]
        xs = r[:,1]
        roi_dict = {}
        roi_dict['top'] = np.min(ys)
        roi_dict['bottom'] = np.max(ys)
        roi_dict['left'] = np.min(xs)
        roi_dict['right'] = np.max(xs)

        roi_list.append(roi_dict)
    return(roi_list)

def make_new_csv_from_rois(zip_file_list, coexpression_by_index_url, corr_count_url, segmented_cells_url):
# INPUT:
#     zip_file_list: list of zip files containing the regions of interest corrected using ImageJ/Fiji
# OUTPUT:
#     return: dictionary of channels (may be removed in the future.  seems unneccessary)
#     creates .csv file of correlations and new roi file

    # dataframe for all baounding box data for all channels of an image
    df_for_csv = pd.DataFrame()
    channel_dictionary = {}
    df = pd.read_csv(segmented_cells_url)

    # for each .zip file of rois
    for f in zip_file_list:

        # read each .zip file and get the name of the channel that file belongs to from the title
        rs = read_imagej_roi_zip(f)
        first_split = f.split('/')
        first_split = first_split[-1]
        split_list = first_split.split('_')
        ch_name = split_list[-2]

        # use channel name to find channel number from the segmented_cells.csv dataframe
        coresponding_ch_number = df[df.channel_name == str(ch_name)]
        coresponding_ch_number = coresponding_ch_number.channel_number.unique()
        # make sure channel name/number matching worked
        assert len(coresponding_ch_number) == 1
        ch_number = int(coresponding_ch_number)
        channel_dictionary[ch_number] = ch_name

        for r in rs:
            # read rois from dictionary
            x1 = r['left']
            y1 = r['top']
            x2 = r['right']
            y2 = r['bottom']

            # add bounding box data to the pandas dataframe
            df_dic = {"channel_number":ch_number,"channel_name":ch_name,"x1":x1,"x2":x2,"y1":y1,"y2":y2}
            df_for_csv = df_for_csv.append(df_dic, ignore_index=True)

    # convert numerical data from dtype string to dtype int
    df_for_csv[["channel_number","x1","x2","y1","y2"]] = df_for_csv[["channel_number","x1","x2","y1","y2"]].apply(pd.to_numeric)
    df_for_csv = df_for_csv.reset_index(drop=True)

    # find_correlations() calculates bb overlap and writes two .csv files
    corr_index_df = find_correlations(df_for_csv, channel_dictionary, coexpression_by_index_url, corr_count_url)
    return(corr_index_df)
    # return(channel_dictionary)

def iou(bb1, bb2):

    # determine the coordinates of the intersection rectangle
    x_left = max(bb1['x1'], bb2['x1'])
    y_top = max(bb1['y1'], bb2['y1'])
    x_right = min(bb1['x2'], bb2['x2'])
    y_bottom = min(bb1['y2'], bb2['y2'])

    if x_right <= x_left or y_bottom <= y_top:
        return 0.0

    ###Common fxn that i litterally stole from a dude on stack
    assert bb1['x1'] < bb1['x2']
    assert bb1['y1'] < bb1['y2']
    assert bb2['x1'] < bb2['x2']
    assert bb2['y1'] < bb2['y2']

    # The intersection of two axis-aligned bounding boxes is always an
    # axis-aligned bounding box
    intersection_area = (x_right - x_left) * (y_bottom - y_top)

    # compute the area of both AABBs
    bb1_area = (bb1['x2'] - bb1['x1']) * (bb1['y2'] - bb1['y1'])
    bb2_area = (bb2['x2'] - bb2['x1']) * (bb2['y2'] - bb2['y1'])

    # compute the intersection over union by taking the intersection
    # area and dividing it by the sum of prediction + ground-truth
    # areas - the interesection area
    iou = intersection_area / float(bb1_area + bb2_area - intersection_area)
    # print(iou)
    assert iou >= 0.0
    assert iou <= 1.0
    return iou

def find_correlations(multi_ch_df, channel_dictionary, index_file_url, corr_count_url, thresh_iou_val=0.3):
    dict_of_ch=channel_dictionary

    # get channel numbers
    slice_nums = multi_ch_df.channel_number.unique()
    slice_nums.sort()

    # start df object where index and columns are the channel numbers.  Fill df with zeros
    corr_df = pd.DataFrame(index=slice_nums, columns=slice_nums)
    corr_df[:] = 0

    # I HAVE NO IDEA WHAT THIS IS FOR
    corr_index_df = multi_ch_df
    for ch_num_for_df in slice_nums:
        ch_name_for_df = dict_of_ch[ch_num_for_df]
        corr_index_df[ch_name_for_df] = None

    for ch_1 in slice_nums:
        # get values from df for a sepecific channel number
        df_1 = multi_ch_df.loc[multi_ch_df['channel_number'] == ch_1]

        # iterate through bbs for channel 1
        for index1, row1 in df_1.iterrows():
            df1_y1 = row1['y1']
            df1_x1 = row1['x1']
            df1_y2 = row1['y2']
            df1_x2 = row1['x2']
            bb1 = {'x1':df1_x1, 'x2':df1_x2, 'y1':df1_y1, 'y2':df1_y2}
            for ch_2 in slice_nums:
                # get data for a second channel
                df_2 = multi_ch_df.loc[multi_ch_df['channel_number'] == ch_2]
                # iterate through bbs for channel 2
                for index2, row2 in df_2.iterrows():
                    df2_y1 = row2['y1']
                    df2_x1 = row2['x1']
                    df2_y2 = row2['y2']
                    df2_x2 = row2['x2']
                    bb2 = {'x1':df2_x1, 'x2':df2_x2, 'y1':df2_y1, 'y2':df2_y2}

                    # does that bb from channel 1 and 2 overlap?  iou() returns a value between 0 and 1 for overlap
                    iou_val = iou(bb1, bb2)
                    # is the overlap greater than the threshold for overlap?
                    if iou_val > thresh_iou_val:
                        # if bbs overlap, add 1 to the correlation df
                        add_df_val = corr_df.at[ch_1, ch_2] + 1
                        corr_df.at[ch_1, ch_2] = add_df_val

                        # add to the co-expression dataframe.  a df where all the bbs are listed by index, and each bb
                        # has a row indicating the gene it overlaps with and the index of the specific bb it overlaps with
                        co_expressor_name = dict_of_ch[ch_2]
                        co_expressor_prev_val = corr_index_df.at[index1, co_expressor_name]
                        if co_expressor_prev_val == None:
                            index_list = [index2]
                            corr_index_df.at[index1, co_expressor_name]=index_list
                        else:
                            co_expressor_prev_val.append(index2)
                            corr_index_df.at[index1, co_expressor_name]=co_expressor_prev_val

    # replace channel number with channel names in the corr_df dataframe
    corr_df = corr_df.rename(columns=dict_of_ch, index=dict_of_ch)

    # print(corr_index_df)
    print(corr_df)

    # save the corr_df and corr_index_df as .csv
    corr_index_df.to_csv(index_file_url, encoding='utf-8', index=True)
    corr_df.to_csv(corr_count_url, encoding='utf-8', index=True)
    return(corr_index_df)

def show_bbs(bb_df):
    # INPUT:
    #     bb_df: dataframe of all bbs
    # OUTPUT: bokeh figure for a channel
    # this is just for quick refference of a channel.  Not used normally
    df = bb_df[bb_df.channel_number==4]
    #print(df)
    top = df['y1'].tolist()
    bottom = df['y2'].tolist()
    left = df['x1'].tolist()
    right = df['x2'].tolist()

    neg_top = [-x for x in top]
    neg_bottom = [-x for x in bottom]

    p = figure()
    p.quad(left=left, right=right, top=neg_top, bottom=neg_bottom)
    show(p)


from django.contrib import admin
from django.template.response import TemplateResponse

from events.models import Event

class CustomAdminSite(admin.AdminSite):
    index_template = "admin/custom-index.html"

    def index(self, request, extra_context=None):
        if request.method == "POST":
            current_event = request.POST.dict().get("event_name", "all")
            request.session["current_event"] = current_event
        else:
            current_event = request.session.get("current_event", "all")

        event_names = Event.read_objects.choices()
        for id, name in event_names:
            if current_event == id:
                self.site_header = name
                break

        app_list = self.get_app_list(request)

        context = {
            **self.each_context(request),
            "title": self.index_title,
            "subtitle": None,
            "app_list": app_list,
            **(extra_context or {}),
            "event_names": event_names,
            "current_event": current_event,
        }

        request.current_app = self.name

        return TemplateResponse(
            request, self.index_template or "admin/index.html", context
        )


#! /usr/bin/env python3

"""
Sækir dagskrárskjöl RÚV og geymir
"""

import requests
import github3
import datetime
import os

GITHUB_USER = "pallih"
GITHUB_REPO = "ruvdagskra"
GITHUB_TOKEN = os.getenv("GITHUB_TOKEN")

CHANNELS = ["ruv", "ruv2", "ras1", "ras2"]
FILETYPE = "xml"
BASEURL = "http://muninn.ruv.is/files/{}/{}/{}-{}-{}/"
DATA_FOLDER = "data"

def save_file_and_commit(channel, year, month, day, file_extension, content):
    gh = github3.login(token=GITHUB_TOKEN)
    repo = gh.repository(GITHUB_USER, GITHUB_REPO)
    folder_path = os.path.join(DATA_FOLDER, year, month, day)
    os.makedirs(folder_path, exist_ok=True)
    filename = "{}-{}-{}-{}{}".format(channel, year, month, day, file_extension)
    filepath = os.path.join(folder_path, filename)
    with open(filepath, "w") as f:
        f.write(content)
    message = "Added: {}".format(filename)
    commit = repo.create_file(filepath, message, content.encode('utf-8'))

def download_and_save(missing_files):
    saved = []
    for missing_file in missing_files:
        filename, file_extension = os.path.splitext(missing_file)
        channel, year, month, day = filename.split("-")
        r = requests.get(BASEURL.format(file_extension.replace(".", ""), channel, year, month, day))

        if r.status_code is not 404:
            save_file_and_commit(channel, year, month, day, file_extension, r.text)
            saved.append(missing_file)
    return saved

def find_missing_files():
    base = datetime.date.today()
    date_list = [base - datetime.timedelta(days=x) for x in range(0, 91)]
    expected = []
    for date in date_list:
        for channel in CHANNELS:
            expected.append("{}-{}-{}-{}.{}".format(channel, date.year, '{:02d}'.format(date.month), '{:02d}'.format(date.day), FILETYPE))
    os.makedirs(DATA_FOLDER, exist_ok=True)
    already_exists = []
    for dirpath, dirnames, filenames in os.walk(DATA_FOLDER):
        for filename in [f for f in filenames if f.endswith(".xml")]:
            already_exists.append(filename)
    missing = [item for item in expected if item not in already_exists]
    return missing

if __name__ == '__main__':
    missing_files = find_missing_files()
    if len(missing_files) is not 0:
        saved = download_and_save(missing_files)
        if len(saved) is not 0:
            message = "Added: {}".format(", ".join(saved))


# -*- coding: utf-8 -*-
import scrapy
import os
from items import HypebeastItem
import datetime
import re
from sneakers import sneaker

class HypebeastSneakerSpider(scrapy.Spider):
    name = 'hypebeast_sneaker'
    page_crawled = 0
    # allowed_domains = ['https://hypebeast.com/footwear']
    start_urls = ['https://hypebeast.com/footwear/page/']
    urls = [('hypebeast','https://hypebeast.com/footwear/page/'), ('snkr_news','https://sneakernews.com/page/')]
    last_ids = {'hype': '-1', 'snkr': '-1','where2b': '-1'}
    last_ids_reached = {'hype': 0, 'snkr': 0, 'where2b': 0}
    most_ids = {'hype': '-1', 'snkr': '-1', 'where2b': '-1'}
    _Debug = False

    def parse(self, response):
        self.last_ids = self.get_last_crawled()
        for (site, url) in self.urls:
            for page_num in range(4,0,-1):
                crawling_url = url + str(page_num)
                if(site == 'hypebeast'):
                    pass
                    #if not(self.last_ids_reached['hype']):
                        #yield scrapy.Request(crawling_url, callback=self.parse_hype_pages)
                else:
                    if not (self.last_ids_reached['snkr']):
                        yield scrapy.Request(crawling_url, callback=self.parse_snkr_pages)

    def parse_hype_pages(self, response):
        if(self.last_ids_reached['hype']):
            return
        item = HypebeastItem()
        selector = scrapy.Selector(response)
        articles = selector.xpath('//div[@class="post-box-content-container"]') ##找到每篇文章的div#
        for article in articles:

            title_div = article.xpath('div[@class="post-box-content-title"]')
            artical_mata = article.xpath('div[@class="post-box-content-meta"]')

            artical_url = title_div.xpath('a/@href').extract()[0]

            if str(artical_url) == self.last_ids['hype'].strip('\n'):
                self.last_ids_reached['hype'] = 1
                return

            if self.most_ids['hype'] == '-1':
                self.most_ids['hype'] = artical_url
            title_text = title_div.xpath('a/@title').extract()[0]

            artical_mata = artical_mata.xpath('div[@class="post-box-content-meta-time-info"]')[0]
            datetime = artical_mata.xpath('span[@class="time"]/time/text()').extract()[0]
            views = artical_mata.xpath('div[@class="post-box-stats"]/hype-count/span[@class="hype-count"]/text()').extract()[0]
            views = views.replace("\n", "")
            views = views.replace(" ", "")
            views = views.replace(",", "")
            views = views.strip('Hypes')
            views = int(views)

            self.page_crawled += 1

            item['url'] = artical_url
            item['title'] = title_text
            item['views'] = views
            item['time'] = self.convert_hype_time(datetime)

            yield item

    def parse_snkr_pages(self, response):
        if(self.last_ids_reached['snkr']):
            return

        selector = scrapy.Selector(response)
        articles = selector.xpath('//div[contains(@class, "-post-box")]')
        title_div = articles.xpath('//div[contains(@class, "post-content")]')

        post_box = title_div[0]
        big_artical_url = post_box.xpath('//h2/a/@href').extract()
        article_url = big_artical_url + post_box.xpath('//h4/a/@href').extract()

        for url in article_url:
            if url == self.last_ids['snkr'].strip('\n'):
                self.last_ids_reached['snkr'] = 1
                return
            yield scrapy.Request(url, callback=self.process_snkr_page)

    def process_snkr_page(self, response):
        if str(response.url) == self.last_ids['hype'].strip('\n'):
            self.last_ids_reached['hype'] = 1
            return
        if (self.last_ids_reached['snkr']):
            return
        print("\n================================Getting SNKRS page:{}==========================\n".format(response.url))
        item = HypebeastItem()
        snkrs = ""
        time = response.url[24:34]
        if (self.is_today(time) and self.most_ids['snkr'] == '-1'):
            self.most_ids['snkr'] = response.url
            self.write_last_crawled(self.most_ids)
        selector = scrapy.Selector(response)
        title = selector.xpath('/html/body/div[1]/div[2]/div/div[1]/div[1]/h1/text()').extract_first()
        votes = selector.xpath('//div[@class = "vote-box"]/div[@class="post-ratings"]/span/i/text()').extract_first().strip(" ")
        votes = votes.strip("VOTES")
        print('-----------Titel = {}-----------\n'.format(title))
        print('-----------Votes = {}-----------\n'.format(votes))
        print('-----------Time = {}-----------\n'.format(time))
        release_divs = selector.xpath('//blockquote[@class = "releases"]/p')
        print('-----------Found {} Release Divs:{}\n'.format(len(release_divs), release_divs))
        for release_div in release_divs:
            snkr = sneaker()
            prize = "".join(release_div.xpath('text()').extract()).strip("\n$").strip(' ')
            print('-----------Prize = {}-----------\n'.format(prize))
            info = release_div.xpath('strong')
            if len(info) > 1:
                date = info[1].xpath('text()').extract()[0].strip("Release Date: ")
            else:
                date = "unknown"
            name = info[0].xpath('text()').extract()[0]
            info = release_div.xpath('//small/text()').extract()
            color = '-'
            code = '-'
            while(len(info)>0):
                if("Color: "in info[0]):
                    color = info[0].strip("Color: ")
                elif("Style Code: " in info[0]):
                    code = info[0].strip("Style Code: ")
                del(info[0])
            snkr.name(name)
            snkr.color(color)
            snkr.prize(prize)
            snkr.release(date)
            snkr.id(code)
            print(str(snkr))

            snkrs += snkr.snkr_name
            snkrs += ", "

        item['url'] = response.url
        item['time'] = time
        item['votes'] = votes
        item['title'] = title
        item['sneaker'] = str(snkrs)

        self.page_crawled += 1
        yield item

    def get_last_crawled(self):
        last_doc = open("last_doc.txt",'r')
        result = {'hype': '', 'snkr': '', 'where2b': ''}

        if(os.stat('last_doc.txt').st_size==0):
            last_doc.close()
            return result
        for line in last_doc:
            id_entry = line.split(',')
            if id_entry[0] == 'where2b':
                result['where2b'] = id_entry[-1]
            elif id_entry[0] == 'snkr':
                result['snkr'] = id_entry[-1]
            elif id_entry[0] == 'hype':
                result['hype'] = id_entry[-1]

        last_doc.close()
        return result

    def write_last_crawled(self, last_crawled):
        last_doc = open("last_doc.txt",'w')
        for site,url in last_crawled.items():
            last_doc.write('{},{}\n'.format(site,url))
        last_doc.close()

    def is_today(self, date):

        '''
        Check if the input date is the current date
        :param date: string e.g.: 2018/07/23
        :return: bool
        '''

        if(datetime.datetime.now().strftime("%Y/%m/%d") == date):
            return True
        else:
            return False

    def convert_hype_time(self, text_time):
        result = ''
        if("ago" in text_time):
            reobject = re.match(r'^(\d+) ([\w]+) ago', text_time)
            if('Hr' in reobject.group(2) or 'Min' in reobject.group(2)):
                result = datetime.datetime.now().strftime("%Y/%m/%d")
            if ('day' in reobject.group(2)):
                time_obj = datetime.datetime.today() - datetime.timedelta(days=int(reobject.group(1)))
                result = time_obj.strftime("%Y/%m/%d")
        else:
            time_obj = datetime.datetime.strptime(text_time, '%b %d, %Y')
            result = time_obj.strftime("%Y/%m/%d")
        return result

    def get_inpage_sneaker(self, url):
        print("\n================================Getting Request From({})==========================\n".format(url))
        return scrapy.Request(url, callback=self.process_snkr_page)


import os

from matplotlib import pylab

CHART_DIR = os.path.sep.join([os.path.curdir, 'charts'])

def plot_precision_recall(auc_score, name, phase, precision, recall, label=None):
    pylab.clf()
    pylab.figure(num=None, figsize=(5, 4))
    pylab.grid(True)
    pylab.fill_between(recall, precision)
    pylab.plot(recall, precision, lw=1)
    pylab.xlim([0.0, 1.0])
    pylab.ylim([0.0, 1.0])
    pylab.xlabel('Recall')
    pylab.ylabel('Precision')
    pylab.title('P/R curve (AUC=%0.2f) / %s' % (auc_score, label))
    filename = name.replace(" ", "_")
    pylab.savefig(os.path.join(CHART_DIR, "pr_%s_%s.png" %
                               (filename, phase)), bbox_inches="tight")


import asyncio
import aiosmtplib as asmtp
import logging

from email.mime import text

from er.util import argmethod
from er.util import options

options.define('smtp_host', default='', help='SMTP server')
options.define('smtp_port', default=465, help='SMTP server')
options.define('smtp_user', default='', help='SMTP username')
options.define('smtp_password', default='', help='SMTP password')
options.define('mail_from', default='', help='Mail from')

_logger = logging.getLogger(__name__)

_queue = None
_consumer = None

async def _consume(queue):
  while True:
    msg = await queue.get()
    _logger.info('Sending mail...')
    for i in range(3):
      if i != 0:
        _logger.warn('Retrying...')
      try:
        async with asmtp.SMTP_SSL(hostname=options.smtp_host, port=options.smtp_port) as server:
          await server.ehlo()
          await server.login(options.smtp_user, options.smtp_password)
          await server.sendmail(options.mail_from, msg['To'], msg.as_string())
        _logger.info('Sent.')
        break
      except Exception as e:
        _logger.error('Send mail error: %s', repr(e))
    queue.task_done()

async def _publish(queue, obj):
  await queue.put(obj)

def init():
  global _queue, _consumer
  _queue = asyncio.Queue()
  _consumer = asyncio.ensure_future(_consume(_queue))

@argmethod.wrap
async def send_mail(to: str, subject: str, content: str):
  msg = text.MIMEText(content, _subtype='html', _charset='UTF-8')
  msg['Subject'] = subject
  msg['From'] = options.mail_from
  msg['To'] = to
  await _publish(_queue, msg)

if __name__ == '__main__':
  argmethod.invoke_by_args()


from StringIO import StringIO

from zope.publisher.http import HTTPRequest

_testEnv =  {
    'SERVER_URL':         'http://foobar.com',
    'HTTP_HOST':          'foobar.com',
    'CONTENT_LENGTH':     '0',
    'GATEWAY_INTERFACE':  'Test/1.0',
}

class TestRequest(HTTPRequest):

    def __init__(self, body_instream=None, environ=None, **kw):
        if body_instream is None:
            body_instream = StringIO('')

        env = {}
        env.update(_testEnv)
        if environ: env.update(environ)
        env.update(kw)

        super(TestRequest, self).__init__(body_instream, env)


import enum
import math
from typing import Optional

import semantic_version

import container_service_extension.common.constants.server_constants as server_constants  # noqa: E501
import container_service_extension.common.constants.shared_constants as shared_constants  # noqa: E501
import container_service_extension.common.utils.core_utils as utils
from container_service_extension.config.server_config import ServerConfig
import container_service_extension.rde.models.common_models as common_models
from container_service_extension.server.pks.pks_cache import PksCache

def get_server_runtime_config() -> ServerConfig:
    import container_service_extension.server.service as cse_service
    return cse_service.Service().get_service_config()

def get_rde_version_in_use() -> str:
    """Get the RDE version used by CSE server.

    :return: rde version
    :rtype: str
    """
    # TODO: Currently in many places, this method is used with expected
    #  return type is string. This should be changed to return semantic version
    #  with all dependencies changed as well.
    config = get_server_runtime_config()
    return str(config.get_value_at('service.rde_version_in_use'))

def get_registered_def_entity_type() -> common_models.DefEntityType:
    """Fetch the native cluster entity type loaded during server startup."""
    from container_service_extension.server.service import Service
    return Service().get_native_cluster_entity_type()

def get_pks_cache() -> PksCache:
    from container_service_extension.server.service import Service
    return Service().get_pks_cache()

def is_pks_enabled() -> bool:
    from container_service_extension.server.service import Service
    return Service().is_pks_enabled()

def is_tkg_plus_enabled(config: Optional[ServerConfig] = None) -> bool:
    """
    Check if TKG plus is enabled by the provider in the config.

    :param ServerConfig config: configuration provided by the user.

    :return: whether TKG+ is enabled or not.
    :rtype: bool
    """
    if not config:
        try:
            config = get_server_runtime_config()
        except Exception:
            return False
    try:
        tkg_plus_enabled = config.get_value_at('service.enable_tkg_plus')
    except KeyError:
        return False
    if isinstance(tkg_plus_enabled, bool):
        return tkg_plus_enabled
    elif isinstance(tkg_plus_enabled, str):
        return utils.str_to_bool(tkg_plus_enabled)
    return False

def should_use_mqtt_protocol(config: ServerConfig) -> bool:
    """Return true if should use the mqtt protocol; false otherwise.

    The MQTT protocol should be used if the config file contains "mqtt" key
        and the CSE server is not being run in legacy mode.

    :param dict config: config yaml file as a dictionary

    :return: whether to use the mqtt protocol
    :rtype: bool
    """
    try:
        has_mqtt_section = config.get_value_at('mqtt') is not None
        running_in_legacy_mode = utils.str_to_bool(config.get_value_at('service.legacy_mode'))  # noqa: E501
        return has_mqtt_section and not running_in_legacy_mode
    except KeyError:
        return False

def is_no_vc_communication_mode(config: Optional[ServerConfig] = None) -> bool:  # noqa: E501
    """Check if TKGm only mode is enabled by the provider in the config.

    :param ServerConfig config: configuration provided by the user.

    :return: whether TKGm only mode is enabled or not.
    :rtype: bool
    """
    if not config:
        try:
            config = get_server_runtime_config()
        except Exception:
            return False
    try:
        is_no_vc_comm = config.get_value_at('service.no_vc_communication_mode')
    except KeyError:
        return False

    if isinstance(is_no_vc_comm, bool):
        return is_no_vc_comm
    elif isinstance(is_no_vc_comm, str):
        return utils.str_to_bool(is_no_vc_comm)
    return False

def is_test_mode(config: Optional[ServerConfig] = None) -> bool:
    """Check if test mode is enabled in the config.

    :param dict config: configuration provided by the user.

    :return: boolean indicating if test mode is enabled.
    :rtype: bool
    """
    if not config:
        try:
            config = get_server_runtime_config()
        except Exception:
            return False
    try:
        config.get_value_at('test')
        return True
    except KeyError:
        return False

def get_template_descriptor_keys(cookbook_version: semantic_version.Version) -> enum.EnumMeta:  # noqa: E501
    """Get template descriptor keys using the cookbook version."""
    # if cookbook version is None, use version 1.0
    if not cookbook_version:
        cookbook_version = \
            server_constants.RemoteTemplateCookbookVersion.Version1.value
    cookbook_version_to_template_descriptor_keys_map = {
        server_constants.RemoteTemplateCookbookVersion.Version1.value: server_constants.RemoteTemplateKeyV1,  # noqa: E501
        server_constants.RemoteTemplateCookbookVersion.Version2.value: server_constants.RemoteTemplateKeyV2  # noqa: E501
    }
    return cookbook_version_to_template_descriptor_keys_map[cookbook_version]

def construct_paginated_response(
        values,
        result_total,
        page_number=shared_constants.CSE_PAGINATION_FIRST_PAGE_NUMBER,
        page_size=shared_constants.CSE_PAGINATION_DEFAULT_PAGE_SIZE,
        page_count=None,
        next_page_uri=None,
        prev_page_uri=None
):
    if not page_count:
        extra_page = 1 if bool(result_total % page_size) else 0
        page_count = result_total // page_size + extra_page
    resp = {
        shared_constants.PaginationKey.RESULT_TOTAL: result_total,
        shared_constants.PaginationKey.PAGE_COUNT: page_count,
        shared_constants.PaginationKey.PAGE_NUMBER: page_number,
        shared_constants.PaginationKey.PAGE_SIZE: page_size,
        shared_constants.PaginationKey.NEXT_PAGE_URI: next_page_uri,
        shared_constants.PaginationKey.PREV_PAGE_URI: prev_page_uri,
        shared_constants.PaginationKey.VALUES: values
    }

    # Conditionally deleting instead of conditionally adding the entry
    # maintains the order for the response
    if not prev_page_uri:
        del resp[shared_constants.PaginationKey.PREV_PAGE_URI]
    if not next_page_uri:
        del resp[shared_constants.PaginationKey.NEXT_PAGE_URI]
    return resp

def create_links_and_construct_paginated_result(
        base_uri,
        values,
        result_total,
        page_number=shared_constants.CSE_PAGINATION_FIRST_PAGE_NUMBER,
        page_size=shared_constants.CSE_PAGINATION_DEFAULT_PAGE_SIZE,
        query_params=None
):
    if query_params is None:
        query_params = {}
    next_page_uri: str = ''
    if 0 < page_number * page_size < result_total:
        # TODO find a way to get the initial url part
        # ideally the request details should be passed down to each of the
        # handler functions as request context
        next_page_uri = f"{base_uri}?page={page_number+1}&pageSize={page_size}"
        for q in query_params.keys():
            next_page_uri += f"&{q}={query_params[q]}"

    page_count = math.ceil(result_total / page_size)
    prev_page_uri: str = ''
    if page_count >= page_number > 1:
        prev_page_uri = f"{base_uri}?page={page_number-1}&pageSize={page_size}"

    # add the rest of the query parameters
    for q in query_params.keys():
        if next_page_uri:
            next_page_uri += f"&{q}={query_params[q]}"
        if prev_page_uri:
            prev_page_uri += f"&{q}={query_params[q]}"

    return construct_paginated_response(values=values,
                                        result_total=result_total,
                                        page_number=page_number,
                                        page_size=page_size,
                                        page_count=page_count,
                                        next_page_uri=next_page_uri,
                                        prev_page_uri=prev_page_uri)


from typing import Any, Dict, Optional
import logging
import traceback

from abcnet import settings
from abcnet.auth import MessageAuthenticator, NoMessageAuthenticator
from abcnet.handlers import MessageDelegator, MessageHandler
from abcnet.networking import SocketHandler, SimpleMD1, MessageDissemination, ContactBook, NetMaintainer
from abcnet.outch import OutputChannel
from abcnet.structures import Message, PeerContactQualifier, PeerContactInfo, MsgType, SocketBinding, Contact
from abcnet.settings import RuntimeSetting
from abcnet.timer import SimpleTimer

logger = logging.getLogger(__name__)

class ChannelLogger:
    """
    This module wraps around a logger and prepends peer id to every log message.
    It is used to distinguish log messages from different service channels from each other
    if they are started in the same process.
    """

    def __init__(self, my_contactinfo: PeerContactQualifier, inner_logger=logger, ):
        """
        :param my_contactinfo: contact information of the peer
        :param inner_logger:  the inner logger that is called for every log invocation. By default the module logger is used.
        """
        self.my_contactinfo = my_contactinfo
        self.inner_logger = inner_logger

    def _pre_id(self, msg):
        """
        Prepends peer id in-front of the given message and returns it as a string.
        """
        return "Node {} - {}".format(self.my_contactinfo, msg)

    def debug(self, msg, *args, **kwargs):
        self.inner_logger.debug(self._pre_id(msg), *args, **kwargs)

    def info(self, msg, *args, **kwargs):
        self.inner_logger.info(self._pre_id(msg), *args, **kwargs)

    def warning(self, msg, *args, **kwargs):
        self.inner_logger.warning(self._pre_id(msg), *args, **kwargs)

    def error(self, msg, *args, **kwargs):
        self.inner_logger.error(self._pre_id(msg), *args, **kwargs)

    def exception(self, msg, *args, exc_info=True, **kwargs):
        self.inner_logger.exception(self._pre_id(msg), exc_info=exc_info, *args, **kwargs)

    def critical(self, msg, *args, **kwargs):
        self.inner_logger.critical(self._pre_id(msg), *args, **kwargs)

class ChannelService:
    """
    A ChannelService manages the sockets necessary to send, broadcast and receive messages to and from peers.
    An instance of ChannelService is handed to each application in the perform maintenance method,
    to be used to contact peers in the network.
    It contains a netmaintainer instance that is responsible for the network topology.

    The most important methods are broadcast_channel and direct_channel.
    """

    def __init__(self, cb: ContactBook,
                 bind_info: SocketBinding,
                 sh: SocketHandler,
                 md: MessageDissemination,
                 ma: MessageAuthenticator,
                 nm: NetMaintainer,
                 magic_number: int):
        """
        Initializes the service by binding zeroMQ sockets.

        :param cb: Contact book instance of this service.
        :type cb: ContactBook
        :param bind_info: Socket binding info of this service.
        :type bind_info: SocketBinding
        :param ma: The authenticator implementation that is used to sign outgoing messages.
        :type ma: MessageAuthenticator
        """
        self.clog = ChannelLogger(cb.self_contact)
        self.__cb: ContactBook = cb
        self.__bind_info: SocketBinding = bind_info
        self.__sh: SocketHandler = sh
        self.__md: MessageDissemination = md
        self.__ma: MessageAuthenticator = ma
        self.__nm: NetMaintainer = nm

        self.magic_number: int = magic_number

    @staticmethod
    def initialize_service(cb: ContactBook,
                           bind_info: SocketBinding,
                           md: MessageDissemination = None,
                           ma: MessageAuthenticator = None) -> "ChannelService":
        """
        Initializes the service with default components.

        :param cb: Contact book instance of this service.
        :type cb: ContactBook
        :param bind_info: Socket binding info of this service.
        :type bind_info: SocketBinding
        :param md: Message dissemination strategy.
        :param ma: The authenticator implementation that is used to sign outgoing messages.
        :type ma: MessageAuthenticator
        """
        if cb is None or not isinstance(cb, ContactBook):
            raise ValueError("Contact book argument is not valid.")
        if bind_info is None and cb.self_contact is not None:
            bind_info = SocketBinding(publish_addr=cb.self_contact.publish_addr, direct_receive_addr=cb.self_contact.receive_addr)
        if not isinstance(bind_info, SocketBinding):
            raise ValueError("binding info is not valid.")

        if ma is None:
            logger.warning("Channel service has no message authenticator. "
                              "Outbound messages will not be authenticated.")
            ma = NoMessageAuthenticator()
            if cb.self_contact and cb.self_contact.public_key:
                raise ValueError("No authenticator was specified that can sign messages. "
                                 "The self contact however has a public key specified.")

        sh = SocketHandler(bind_info)

        if md is None:
            md: MessageDissemination = SimpleMD1(cb, sh)

        magic_nr: int = settings.RuntimeSetting.MAGIC_NUMBER

        from abcnet.networking import CliqueNetMaintainer
        nm = CliqueNetMaintainer(cb, sh)

        return ChannelService(cb, bind_info, sh, md, ma, nm, magic_nr)

    @property
    def contact(self):
        """
        Returns the self contact information of this peer.
        """
        return self.__cb.self_contact

    @property
    def contacts(self) -> ContactBook:
        """
        Returns the contact book of network peers.
        """
        return self.__cb

    @property
    def net_maintainer(self) -> NetMaintainer:
        """
        Returns the net maintainer implementation.
        """
        return self.__nm

    def poll_msg(self, timeout: float = 0.01) -> Optional[Message]:
        """
        Polls the next message from the receive sockets, waiting the given timeout if necessary.
        If no message was present in the given time, None is returned.
        polling message consumed them, and there is no way to requeue messages.
        That is why this method should only be used by a single MessageDelegator.

        :param timeout: Time in seconds to wait for the next message.
        :type timeout: float
        :return: Optionally a newly received message.
        :rtype: Optional[Message]
        """
        poller = self.__sh.msg_poller()
        msg: Optional[Message] = poller(timeout)
        return msg

    def broadcast_channel(self) -> OutputChannel:
        """
        Creates and returns an OutputChannel, that passes messages as broadcast to the entire network.

        :return: Broadcast OutputChannel
        :rtype: OutputChannel
        """
        return OutputChannel(self.__ma, self.__md.broadcast())

    def direct_channel(self, peer: PeerContactInfo) -> OutputChannel:
        """
        Creates and returns an OutputChannel, that passes messages directly to the given peer.
        Sent messages are not encrypted and could potent

        :param peer: Peer to be directly contacted.
        :type peer: PeerContactInfo
        :return: Direct OutputChannel
        :rtype: OutputChannel
        """
        if peer in self.__cb:
            self.__cb.get_peer(peer).log_outgoing_contact()
        msg_sender = self.__md.direct(peer)
        return OutputChannel(self.__ma, msg_sender)

    def shutdown(self):
        """
        Closes the sockets used by this ChannelService.
        After shutdown is called, this instance is unusable.
        This method does not block, but the underlying sockets may not close immediately.
        :return: None
        :rtype: None
        """
        self.__sh.close_all()

class BaseApp:
    """
    A Base Application instance represents a single network peer application stack.
    Each BaseApp controls a single ChannelService instance.
    There is a 1-to-1-relation between BaseApp and ChannelService.
    In the `step` method, it reads the messages from the input sockets and delegates them to the message handlers.
    """

    _handlers: Dict[str, MessageHandler]

    _msg_delegator: Optional[MessageDelegator]

    def __init__(self, cs: ChannelService, handlers: Dict[str, MessageHandler] = None):
        """
        Initializes the base application given its channel service and initial message handlers.
        Message handlers can also be registered after the baseapp has been initialized.
        The given handlers
        :param cs: Channel Service to be used.
        :type cs: ChannelService
        :param handlers: list of initial message handlers
        :type handlers: MessageHandler
        """
        self.cs = cs
        self._handlers = {}
        self._msg_delegator = None
        if handlers:
            for app_name in handlers:
                if isinstance(handlers[app_name], MessageHandler):
                    self.register_app_layer(app_name, handlers[app_name])

    def register_app_layer(self, app_name:str,  app_layer: MessageHandler):
        """
        Registers the application under the given name.
        """
        if app_name not in self._handlers:
            logger.debug("Added new app layer: %s", app_name)
        else:
            logger.info("App replaced: %s", app_name)
        self._handlers[app_name] = app_layer
        del self._msg_delegator
        self._msg_delegator = None

    def app(self, app_name: str) -> Any:
        """
        Returns the app that was registered under the given name.
        """
        return self._handlers[app_name]

    @property
    def msg_del(self):
        """
        Returns the message delegator.
        """
        if self._msg_delegator is None:
            self._msg_delegator = MessageDelegator(self._handlers.values())
        return self._msg_delegator

    def maintain(self, force_maintenance=False):
        """
        Performs a maintenance of all registered applications.
        """
        for app_name in self._handlers:
            app = self._handlers[app_name]
            timer = RuntimeSetting.APP_LAYER_MAINTENANCE_TIMEOUT_WARN_LOG.stop_timer(start=True)
            try:
                maintenance_performed = False
                try:
                    app.perform_maintenance(self.cs, force_maintenance=force_maintenance)
                    maintenance_performed = True
                except TypeError as e:
                    if str(e) != 'perform_maintenance() got an unexpected keyword argument \'force_maintenance\'':
                        raise
                if not maintenance_performed:
                    app.perform_maintenance(self.cs)
            except Exception as e:
                logger.error("Error performing maintenance in %s.", app_name, exc_info=True)

            if timer.check():
                logger.warning("App maintenance of handler %s took too long: %s", app_name, timer.run_time_str())

    def handle_next_msg(self, timeout: float) -> bool:
        """
        Handles a single message that can be polled in the given time.
        """
        return self.msg_del.delegate_next_msg(timeout, self.cs)

    def handle_remaining_messages(self, timeout: float = 0):
        """
        Pulls messages until timeout and handles each message sequentially.
        """
        timer = SimpleTimer(timeout, start=True)
        message_remain = True
        while message_remain:
            message_remain = self.handle_next_msg(timer.remaining_time)

    def step(self, timeout: float):
        """
        Performs a step.
        A step consists of a perform maintenance round and handling the remaining input queue.
        """
        self.maintain()
        self.handle_remaining_messages(timeout)

    def close(self):
        """
        Closes the application, shutting down all apps.
        """
        for app in self._handlers.values():
            try:
                app.close()
            except Exception:
                logger.warning("Exception while closing app: {}", app)
                traceback.print_exc()
        self.cs.shutdown()