import xml.etree.ElementTree as xml
import json
import argparse
import re
import copy

def build_parser():
    parser = argparse.ArgumentParser()
    parser.add_argument('-xml', '--xml-file', type=str, required=True)
    parser.add_argument('-out', '--out-file', type=str, required=True)
    group1 = parser.add_mutually_exclusive_group()
    group1.add_argument('-dets-only', '--detections-only', action='store_true')
    group1.add_argument('-imgs-info', '--images-info', action='store_true')
    parser.add_argument('-shortened-file-names', '--shortened-file-names', action='store_true')
    return parser

def cvat_segmentation_to_coco(cvat_segmentation):
    coco_segmentation = [[float(x) for x in re.split(',|;', cvat_segmentation)]]
    return coco_segmentation

def segmentation_to_bbox(segmentation):  # json segmentation
    segmentation = segmentation[0]
    x = segmentation[0::2]
    y = segmentation[1::2]
    bbox = [min(x), min(y), max(x)-min(x), max(y)-min(y)]
    return bbox

def shorten_file_names(images_to_shorten):
    images = copy.deepcopy(images_to_shorten)
    if len(images) == 0:
        return images
    if len(images) == 1:
        images[0]['file_name'] = images[0]['file_name'].split('/')[-1]
        return images
    splitted_file_name = images[0]['file_name'].split('/')
    shortening_deep = len(splitted_file_name)-1
    if shortening_deep == 0:
        return
    for image in images[1:]:
        splitted_prev_file_name = splitted_file_name
        splitted_file_name = image['file_name'].split('/')
        for i in range(shortening_deep):
            if splitted_file_name[i] != splitted_prev_file_name[i]:
                shortening_deep = i
                if shortening_deep == 0:
                    return images
                break
    for image in images:
        image['file_name'] = '/'.join(image['file_name'].split('/')[shortening_deep:])
    return images

def cvat_root2coco_dict(root, detections_only=False, images_info=False, shortened_file_names=False):
    assert (not detections_only) or (not images_info)
    images = []
    annotations = []
    categories = []

    # get categories
    labels = root.find('meta').find('task').find('labels')
    category_name_to_id = {}
    cat_id = 1
    for label in labels:
        category = {"id": cat_id, "name": label[0].text}
        categories.append(category)
        category_name_to_id[label.find('name').text] = cat_id
        cat_id += 1

    # get images and annotations
    ann_id = 1
    for cvat_image in root.findall('image'):
        image = dict()
        image["id"] = int(cvat_image.attrib["id"])
        image["height"] = int(cvat_image.attrib["height"])
        image["width"] = int(cvat_image.attrib["width"])
        image["file_name"] = cvat_image.attrib.pop("name")
        images.append(image)
        for box in cvat_image.findall('box'):
            annotation = dict()
            bbox = [float(box.attrib["xtl"]), float(box.attrib["ytl"]),
                    float(box.attrib["xbr"]) - float(box.attrib["xtl"]),
                    float(box.attrib["ybr"]) - float(box.attrib["ytl"])]
            annotation["area"] = bbox[2] * bbox[3]
            annotation["iscrowd"] = 0
            annotation["image_id"] = int(cvat_image.attrib["id"])
            annotation["bbox"] = bbox
            annotation["category_id"] = category_name_to_id[box.attrib["label"]]
            annotation["id"] = ann_id
            if "score" in box.attrib.keys():
                annotation["score"] = float(box.attrib["score"])
            annotations.append(annotation)
            ann_id += 1
        for polygon in cvat_image.findall('polygon'):
            annotation = dict()
            annotation["iscrowd"] = 0
            annotation["image_id"] = int(cvat_image.attrib["id"])
            annotation["segmentation"] = cvat_segmentation_to_coco(polygon.attrib["points"])
            annotation["bbox"] = segmentation_to_bbox(annotation["segmentation"])
            annotation["area"] = annotation["bbox"][2] * annotation["bbox"][3]
            annotation["category_id"] = category_name_to_id[polygon.attrib["label"]]
            annotation["id"] = ann_id
            if "score" in polygon.attrib.keys():
                annotation["score"] = float(polygon.attrib["score"])
            annotations.append(annotation)
            ann_id += 1

    if shortened_file_names:
        images = shorten_file_names(images)
    if detections_only:
        json_dict = {annotations}
    elif images_info:
        json_dict = {'images': images, 'categories': categories}
    else:
        json_dict = {'images': images, 'annotations': annotations, 'categories': categories}
    return json_dict

def cvat2coco(xml_file, out_file, detections_only=False, images_info=False, shortened_file_names=False):
    tree = xml.parse(xml_file)
    root = tree.getroot()
    json_dict = cvat_root2coco_dict(root, detections_only, images_info, shortened_file_names)
    with open(out_file, 'w') as f:
        json.dump(json_dict, f, indent=2)

if __name__ == '__main__':
    parser = build_parser()
    args = parser.parse_args()
    cvat2coco(**vars(args))


#-------------------------------------------------------------------------------
# Copyright (c) 2012 Gael Honorez.
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the GNU Public License v3.0
# which accompanies this distribution, and is available at
# http://www.gnu.org/licenses/gpl.html
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#-------------------------------------------------------------------------------

from PyQt4 import QtCore, QtGui
from PyQt4 import QtWebKit
from stat import *
import util
import urllib
import logging
import os
from fa import maps
from vault import luaparser
import urllib2
import re
import json

logger = logging.getLogger("faf.vault")
logger.setLevel(logging.DEBUG)

class MapVault(QtCore.QObject):
    def __init__(self, client, *args, **kwargs):
        QtCore.QObject.__init__(self, *args, **kwargs)
        self.client = client

        logger.debug("Map Vault tab instantiating")

        self.ui = QtWebKit.QWebView()
        self.ui.page().mainFrame().javaScriptWindowObjectCleared.connect(self.addScript)

        self.client.mapsTab.layout().addWidget(self.ui)

        self.loaded = False
        self.client.showMaps.connect(self.reloadView)
        self.ui.loadFinished.connect(self.ui.show)
        self.reloadView()

    @QtCore.pyqtSlot()
    def reloadView(self):
        if (self.loaded):
            return
        self.loaded = True

        self.ui.setVisible(False)

        #If a local theme CSS exists, skin the WebView with it
        if util.themeurl("vault/style.css"):
            self.ui.settings().setUserStyleSheetUrl(util.themeurl("vault/style.css"))

        self.ui.setUrl(QtCore.QUrl("http://www.faforever.com/faf/vault/maps.php?username={user}&pwdhash={pwdhash}".format(user=self.client.login, pwdhash=self.client.password)))

    @QtCore.pyqtSlot()
    def addScript(self):
        frame  = self.ui.page().mainFrame()
        frame.addToJavaScriptWindowObject("webVault", self)

    def __preparePositions(self, positions, map_size):
        img_size = [256, 256]
        size = [int(map_size['0']), int(map_size['1'])]
        off_x = 0
        off_y = 0

        if (size[1] > size[0]):
            img_size[0] = img_size[0]/2
            off_x = img_size[0]/2
        elif (size[0] > size[1]):
            img_size[1] = img_size[1]/2
            off_y = img_size[1]/2

        cf_x = size[0]/img_size[0]
        cf_y = size[1]/img_size[1]

        regexp = re.compile(" \d+\.\d*| \d+")

        for postype in positions:
            for pos in positions[postype]:
                string = positions[postype][pos]
                values = regexp.findall(positions[postype][pos])
                x = off_x + float(values[0].strip())/cf_x
                y = off_y + float(values[2].strip())/cf_y
                positions[postype][pos] = [int(x), int(y)]

    @QtCore.pyqtSlot()
    def uploadMap(self):

        mapDir = QtGui.QFileDialog.getExistingDirectory (self.client, "Select the map directory to upload", maps.getUserMapsFolder(),  QtGui.QFileDialog.ShowDirsOnly)
        logger.debug("Uploading map from: " + mapDir)
        if mapDir != "" :
            if maps.isMapFolderValid(mapDir) :
                os.chmod(mapDir, S_IWRITE)
                mapName = os.path.basename(mapDir)
                zipName = mapName.lower()+".zip"

                scenariolua = luaparser.luaParser(os.path.join(mapDir,maps.getScenarioFile(mapDir)))
                scenarioInfos = scenariolua.parse({'scenarioinfo>name':'name', 'size':'map_size', 'description':'description', 'count:armies':'max_players','map_version':'version','type':'map_type','teams>0>name':'battle_type'}, {'version':'1'})

                if scenariolua.error:
                    logger.debug("There were " + str(scenariolua.errors) + " errors and " + str(scenariolua.warnings) + " warnings.")
                    logger.debug(scenariolua.errorMsg)
                    QtGui.QMessageBox.critical(self.client, "Lua parsing error", scenariolua.errorMsg + "\nMap uploading cancelled.")
                else:
                    if scenariolua.warning:
                        uploadmap = QtGui.QMessageBox.question(self.client, "Lua parsing warning", scenariolua.errorMsg + "\nDo you want to upload the map?", QtGui.QMessageBox.Yes, QtGui.QMessageBox.No)
                    else:
                        uploadmap = QtGui.QMessageBox.Yes
                    if uploadmap == QtGui.QMessageBox.Yes:
                        savelua = luaparser.luaParser(os.path.join(mapDir,maps.getSaveFile(mapDir)))
                        saveInfos = savelua.parse({'markers>mass*>position':'mass:__parent__', 'markers>hydro*>position':'hydro:__parent__', 'markers>army*>position':'army:__parent__'})
                        if savelua.error or savelua.warning:
                           logger.debug("There were " + str(scenariolua.errors) + " errors and " + str(scenariolua.warnings) + " warnings.")
                           logger.debug(scenariolua.errorMsg)

                        self.__preparePositions(saveInfos, scenarioInfos["map_size"])

                        tmpFile = maps.processMapFolderForUpload(mapDir, saveInfos)
                        if not tmpFile:
                            QtGui.QMessageBox.critical(self.client, "Map uploading error", "Couldn't make previews for " + mapName + "\nMap uploading cancelled.")
                            return None

                        qfile = QtCore.QFile(tmpFile.name)
                        self.client.writeToServer("UPLOAD_MAP", zipName, scenarioInfos, qfile)

                        #removing temporary files
                        qfile.remove()
            else :
                QtGui.QMessageBox.information(self.client,"Map selection",
                        "This folder doesn't contain valid map datas.")

    @QtCore.pyqtSlot(str)
    def downloadMap(self, link):
        link = urllib2.unquote(link)
        name = maps.link2name(link)
        if not maps.isMapAvailable(name):
            maps.downloadMap(name)
            maps.existMaps(True)
        else:
            show = QtGui.QMessageBox.question(self.client, "Already got the Map", "Seems like you already have that map!
Would you like to see it?", QtGui.QMessageBox.Yes, QtGui.QMessageBox.No)
            if show == QtGui.QMessageBox.Yes:
                util.showInExplorer(maps.folderForMap(name))

    @QtCore.pyqtSlot(str)
    def checkMaps(self, data):
        exist_maps = list()
        viewed_maps = json.loads(data)
        for id, name in viewed_maps.iteritems():
            if maps.mapExists(name):
               exist_maps.append(id)
        if len(exist_maps) > 0:
            json_str = json.dumps(exist_maps).replace('"', '\\"')
            self.ui.page().mainFrame().evaluateJavaScript('handle_exist_maps("%s")' % json_str)


from __future__ import division

# LIBTBX_SET_DISPATCHER_NAME diffBragg.shoebx

from pylab import *

from argparse import ArgumentParser
parser = ArgumentParser()
parser.add_argument("modeler_file", type=str, help="path to a diffBragg modeler file (output from hopper, see the imgs folder in the outdir)")
parser.add_argument("--stacked", action="store_true")
parser.add_argument("--model_clim", nargs=2, default=[0,None], type=float)
parser.add_argument("--data_clim", nargs=2, default=[0,None], type=float)
args = parser.parse_args()

FIG,(ax0,ax1,ax2) = subplots(nrows=1,ncols=3)
FIG.set_size_inches((5,2))

def press(event):
    if event.key == 'right':
        FIG.loop_counter += 1
    elif event.key=="left":
        FIG.loop_counter = FIG.loop_counter -1
    FIG.loop_counter = max(FIG.loop_counter,0)
    FIG.loop_counter = min(FIG.loop_counter, FIG.nspots-1)

    if event.key=="escape":
        FIG.loop_counter = FIG.nspots

def centroid_poly(x,y):
    y1 = y[:-1]
    y2 = y[1:]

    x1 = x[:-1]
    x2 = x[1:]
    xy = (x1*y2 - x2*y1)
    Cx = np.sum((x1+x2)*xy)/6.
    Cy = np.sum((y1+y2)*xy)/6.
    A = 0.5*np.sum(xy)
    return Cx/A, Cy/A

M = np.load(args.modeler_file, allow_pickle=True)[()]
num_spots = len(M.pids)

FIG.loop_counter = 0
FIG.nspots = num_spots
FIG.canvas.mpl_connect('key_press_event', press)

assert len(set(M.roi_id)) == max(M.roi_id)+1
sigma_rdout = M.params.refiner.sigma_r / M.params.refiner.adu_per_photon
#M.hi = (0,0,0)]
#hkls = M.Hi
cmap = 'gray_r'

from cctbx import uctbx
# TODO put unit cell manager in the modeler file
#PSII
uc = uctbx.unit_cell((116.92, 221.635, 307.834, 90, 90, 90))
#lysozyme?
uc = uctbx.unit_cell((79.1,79.1,38.4,90,90,90))
diffs = []
d_max = 0
d_min = 9999

#F2 = plt.figure()
#AX2 = F2.gca()
#AX2.add_patch(Rectangle(xy=(0,0), width=4000, height=4000, fc='none', ec='b'))
#AX2.set_xlim(-10,4010)
#AX2.set_ylim(-10,4010)

if args.stacked:

    if isinstance(M.all_sigma_rdout, np.ndarray):
        data_subimg, model_subimg, trusted_subimg, bragg_subimg, sigma_rdout_subimg = M.get_data_model_pairs()
    else:
        data_subimg, model_subimg, trusted_subimg, bragg_subimg = M.get_data_model_pairs()
        sigma_rdout_subimg = None

    sub_sh = tuple(np.max([im.shape for im in model_subimg], axis=0))
    size_edg = int(np.sqrt(len(data_subimg))) + 1
    full_im = np.zeros((size_edg * sub_sh[0], size_edg * sub_sh[1]))
    full_dat_im = np.zeros((size_edg * sub_sh[0], size_edg * sub_sh[1]))
    for j in range(size_edg):
        for i in range(size_edg):
            mod_idx = j * size_edg + i
            if mod_idx >= len(model_subimg):
                continue
            im = model_subimg[mod_idx].copy()
            dat_im = data_subimg[mod_idx].copy()
            ydim, xdim = im.shape
            if im.shape != sub_sh:
                im = np.pad(im, ((0, sub_sh[0] - ydim), (0, sub_sh[1] - xdim)), mode='constant', constant_values=np.nan)
                dat_im = np.pad(dat_im, ((0, sub_sh[0] - ydim), (0, sub_sh[1] - xdim)), mode='constant',
                                constant_values=np.nan)
            Ysl = slice(j * sub_sh[0], (j + 1) * sub_sh[0], 1)
            Xsl = slice(i * sub_sh[1], (i + 1) * sub_sh[1], 1)
            full_im[Ysl, Xsl] = im
            full_dat_im[Ysl, Xsl] = dat_im
    subplot(121)
    gca().set_title("DATA", fontsize=18)
    imshow(full_dat_im, vmin=args.data_clim[0], vmax=args.data_clim[1], cmap='gray_r')
    gca().set_xticks(np.arange(sub_sh[1], size_edg*sub_sh[1], sub_sh[1]))
    gca().set_yticks(np.arange(sub_sh[0], size_edg*sub_sh[0], sub_sh[0]))
    gca().grid(1, color='k', ls='--')
    gca().set_xticklabels([])
    gca().set_yticklabels([])
    gca().tick_params(length=0)
    #gca().set_xlim(0, size_edg*sub_sh[1])
    #gca().set_ylim(0, size_edg*sub_sh[0])
    #gca().set_yticks([])
    subplot(122)
    gca().set_title("MODEL", fontsize=18)
    imshow(full_im, vmin=args.model_clim[0], vmax=args.model_clim[1], cmap='gnuplot')
    gca().set_xticks(np.arange(sub_sh[1], size_edg*sub_sh[1], sub_sh[1]))
    gca().set_yticks(np.arange(sub_sh[0], size_edg*sub_sh[0], sub_sh[0]))
    gca().grid(1, color='w', ls='--')
    gca().set_xticklabels([])
    gca().set_yticklabels([])
    gca().tick_params(length=0)
    #gca().set_xlim(0, size_edg*sub_sh[1])
    #gca().set_ylim(0, size_edg*sub_sh[0])
    #gca().set_xticks([])
    #gca().set_yticks([])
    subplots_adjust(wspace=0, hspace=0, bottom=0.02, top=0.95, left=0.01, right=0.99)
    show()
    exit()

a = b = None
while FIG.loop_counter < num_spots:
    i_h = FIG.loop_counter
    h = k = l = np.nan
    d = -1 #uc.d((h,k,l))
    #if d > d_max: d_max = d
    #if d < d_min: d_min = d

    # you could put a filter here to choose only those refls in a certain resolution
    sel = M.roi_id == i_h
    x1, x2, y1, y2 = M.rois[i_h]
    sh = y2 - y1, x2 - x1
    data = M.all_data[sel].reshape(sh)
    trusted = M.all_trusted[sel].reshape(sh)
    any_trusted = np.any(trusted)

    bg = M.all_background[sel].reshape(sh)
    bragg = M.best_model[sel].reshape(sh)
    model = bragg + bg
    diff = data.sum() - model.sum()

    Z = (model - data) / np.sqrt(model + sigma_rdout ** 2)
    diffs.extend(Z.ravel())

    vals = Z[trusted]
    data_trust = data[trusted]
    if not any_trusted:
        data_thresh = 0
    else:
        data_thresh = np.percentile(data_trust,97)
    y,x = np.indices(data.shape)

    ycent = (bragg.ravel()*y.ravel()).sum() /bragg.sum()
    xcent = (bragg.ravel()*x.ravel()).sum() / bragg.sum()

    if not any_trusted:
        sigZ_val = np.nan
    else:
        sigZ_val = Z[trusted].std()

    for ax in ax0,ax1,ax2:
        ax.clear()
    im = ax0.imshow(data, cmap=cmap)
    vmin, vmax = im.get_clim()
    ax1.imshow(model,cmap=cmap, vmin=vmin, vmax=vmax)

    ax1.contour(bragg, levels=[1e-4,1e-3,1e-2,1e-1,1,10], cmap='jet')
    C = ax0.contour(bragg, levels=[1e-4,1e-3,1e-2,1e-1,1,10], cmap='jet')
    ax1.plot( xcent, ycent, 'rx')
    #C = ax0.contour(bragg, levels=[0.1*data_thresh, 0.5*data_thresh,data_thresh], cmap='jet')
    #from IPython import embed;embed()

    #try:
    #    segs = C.allsegs[-1][0]
    #    xcent, ycent = centroid_poly(segs[:,0], segs[:,1])
    #    ax0.plot( [xcent], [ycent], 'o', mec='b', ms=10, alpha=0.5)
    #    ax0.plot( segs[:,0], segs[:,1], '.', ms=3, color='k')
    #except IndexError:
    #    pass

    v = np.max(np.abs(Z))
    v = 10
    ax2.imshow(Z, cmap='RdYlBu', vmin=-v, vmax=v)

    for AX in [ax0, ax1, ax2]:
        AX.grid(1, color='#777777', ls='--', lw=0.4)
        AX.set_xticklabels([])
        AX.set_yticklabels([])
    ax0.set_title("data %d,%d,%d,%d"% (x1,x2,y1,y2), pad=0, fontsize=8)
    ax1.set_title("model", pad=0, fontsize=8)
    ax2.set_title("Z (sigZ=%f)" % sigZ_val, pad=0, fontsize=8)

    FIG.suptitle("spot %d" % i_h)
    if a is not None:
        a.remove()
    if b is not None:
        b.remove()
    a = FIG.add_axes([.91, .25, 0.02, .5])
    FIG.colorbar(ax2.images[0], cax=a)
    a.tick_params(length=0, labelsize=8, pad=1)
    a.set_title("Z", pad=0, fontsize=8)

    b = FIG.add_axes([.2, .1, .33, .02])
    FIG.colorbar(ax0.images[0], cax=b, orientation='horizontal')
    b.yaxis.set_ticks_position('left')
    b.tick_params(length=0, labelsize=8, pad=5)
    b.set_title("counts", pad=0, fontsize=8)
    draw()
    waitforbuttonpress()

    i_h += 1

plt.close()


from keras.preprocessing import sequence
from sklearn.cluster import KMeans
from sklearn.metrics.pairwise import euclidean_distances
from sklearn.preprocessing import LabelEncoder

import multiprocessing
import os
import pandas

import numpy as np

def get_input_sequences(input_data, padding="avg", return_class_encoder=False):
    classes = sorted([f for f in os.listdir(input_data)
                      if os.path.isdir(os.path.join(input_data, f)) and not f.startswith('.')],
                     key=lambda f: f.lower())
    X = []
    Y = []
    sequence_lengths = []
    for class_name in classes:
        files = sorted([f for f in os.listdir(os.path.join(input_data, class_name))
                        if os.path.isfile(os.path.join(input_data, class_name, f)) and not f.startswith('.')
                        and f.endswith(".csv")], key=lambda f: f.lower())
        for file in files:
            df = pandas.read_csv(os.path.join(input_data, class_name, file))
            values = df.values
            nan_inds = np.where(np.isnan(values))
            values[nan_inds] = 0
            inf_inds = np.where(np.isinf(values))
            values[inf_inds] = np.sign(values[inf_inds])
            X.append(values)
            sequence_lengths.append(len(df.values))
            Y.append(class_name)
    sequence_lengths = np.array(sequence_lengths)
    X = np.array(X)
    if padding == "avg":
        length = int(sequence_lengths.mean())
    elif padding == "max":
        length = max(sequence_lengths)
    elif padding == "min":
        length = min(sequence_lengths)
    else:
        print("No padding selected, returning raw sequences.")
        length = None
    if length is not None:
        X = sequence.pad_sequences(X, maxlen=length, dtype="float64")
    Y = np.array(Y)
    encoder = LabelEncoder()
    encoder.fit(Y)
    Y = encoder.transform(Y)
    if return_class_encoder:
        return X, Y, encoder
    else:
        return X, Y

def kmeans_frame_selection(frames, k=20, seed=0):

    kmeans = KMeans(n_clusters=k, random_state=seed, n_jobs=multiprocessing.cpu_count()).fit(frames)
    centers = kmeans.cluster_centers_
    frame_numbers = []
    for center in centers:
        distances = [distance[0] for distance in euclidean_distances(frames, [center])]
        frame_number = distances.index(min(distances))
        frame_numbers.append(frame_number)
    frame_numbers.sort()

    representative_frames = []
    for i in range(len(frame_numbers)):
        frame = frames[frame_numbers[i]]
        representative_frames.append(frame)

    return np.array(representative_frames), np.array(frame_numbers)

def kmeans_seq_reduction(dataset, k=20, seed=0):

    for stream_idx, stream in enumerate(dataset):
        new_dataset = []
        for instance in stream:
            instance_length = instance.shape[0]
            new_instance, frames = kmeans_frame_selection(instance, min(k, instance_length), seed)
            new_dataset.append(new_instance)
        dataset[stream_idx] = sequence.pad_sequences(new_dataset, maxlen=k, dtype="float64")
    return dataset

def kmeans_sync_seq_reduction(dataset, k=20, seed=0):

    for instance_idx in range(dataset[0].shape[0]):
        view_vectors = []
        for stream in dataset:
            view_vectors.append(stream[instance_idx])
        instance = np.concatenate(tuple(view_vectors), axis=1)
        instance_length = instance.shape[0]
        conc_instance, frames = kmeans_frame_selection(instance, min(k, instance_length), seed)

        for stream_idx in range(len(dataset)):
            dataset[stream_idx][instance_idx] = dataset[stream_idx][instance_idx][frames]
    for idx, stream in enumerate(dataset):
        dataset[idx] = sequence.pad_sequences(stream.tolist(), maxlen=k, dtype="float64")
    return dataset

def multiple_sequence_padding(input_streams, padding="max"):

    lengths = []
    for stream in input_streams:
        length = stream.shape[-2]
        lengths.append(length)
    lengths = np.array(lengths)

    if padding == "avg":
        max_length = int(lengths.mean())
    elif padding == "max":
        max_length = max(lengths)
    elif padding == "min":
        max_length = min(lengths)
    else:
        raise ValueError("padding must be either avg, max or min")

    for idx, x in enumerate(input_streams):
        input_streams[idx] = sequence.pad_sequences(x, maxlen=max_length, dtype="float64")

    return input_streams

def multiple_sequence_padding_means(input_streams, padding="max"):

    lengths = []
    for stream in input_streams:
        for instance in stream:
            length = instance.shape[-2]
            lengths.append(length)
    lengths = np.array(lengths)

    if padding == "avg":
        max_length = int(lengths.mean())
    elif padding == "max":
        max_length = max(lengths)
    elif padding == "min":
        max_length = min(lengths)
    else:
        raise ValueError("padding must be either avg, max or min")

    for stream_idx, stream in enumerate(input_streams):
        input_streams[stream_idx] = kmeans_seq_reduction(stream, k=max_length)

    for idx, x in enumerate(input_streams):
        input_streams[idx] = sequence.pad_sequences(x, maxlen=max_length, dtype="float64")

    return input_streams

def pad_views_sequences(views_list, length):
    padded_list = []
    for view in views_list:
        view_len = view.shape[0]
        turn = 0
        while view_len > length:
            view = np.delete(view, (turn), axis=0)
            view_len = view.shape[0]
            turn = (turn + 1) * -1
        padded_list.append(view)
    return padded_list

def synchronize_views(input_streams):

    num_instances = input_streams[0].shape[0]

    for instance_idx in range(num_instances):

        seq_lengths = []
        samples = []
        for view in input_streams:
            seq_lengths.append(view[instance_idx].shape[0])
            samples.append(view[instance_idx])
        length = min(seq_lengths)
        padded_instances = pad_views_sequences(samples, length)

        for view_idx in range(len(padded_instances)):
            input_streams[view_idx][instance_idx] = padded_instances[view_idx]

    return input_streams


"""
app.py
Final version
Updated January 18th, 2022

Written by: Kai Wang
"""

""" -----imports------"""
from flask import Flask, render_template, request, redirect
from flask_mysqldb import MySQL
import csv
import MySQLdb

app = Flask(__name__)

""" -----MySQL connection setup-----
	Change the password to the
	password of your system 		"""

app.config['MYSQL_HOST'] = 'localhost'
app.config['MYSQL_USER'] = 'root'
app.config['MYSQL_PORT'] = 3306

app.config['MYSQL_PASSWORD'] = ""
app.config['MYSQL_DB'] = "shopify_inventory"
app.config['MYSQL_CURSORCLASS'] = 'DictCursor'

mysql = MySQL(app)

""" ---- MainMenu API ----- """
@app.route('/', methods=['GET', 'POST'])
def mainmenu():
    if request.method == 'POST':
        if request.form['submit_button'] == 'Create':
            return redirect('/create')
        elif request.form['submit_button'] == 'View':
            return redirect('/view')

    return render_template('mainmenu.html')

""" --- Create API ----- """
@app.route('/create', methods=['GET', 'POST'])
def create():
    if request.method == 'POST':
        if request.form['submit_button'] == 'Create':
        # Fetch form data
            userDetails = request.form
            name = userDetails['name']
            price = userDetails['price']
            department = userDetails['department']
            description = userDetails['description']
            store_URL = userDetails['store_URL']

            #Insert(Create) query
            cur = mysql.connection.cursor()
            cur.execute("INSERT INTO inventory(name,price,department,description,store_URL) VALUES (%s, %s,%s,%s,%s)", (name,price,department,description,store_URL))
            mysql.connection.commit()
            cur.close()

        elif request.form['submit_button'] == 'View':
            return redirect('/view')

        elif request.form['submit_button'] == 'Return':
            return redirect('/')

    return render_template('create.html')

""" ---- Edit API ----- """
@app.route('/edit',methods=['GET', 'POST'])
def edit():
    if request.method == 'POST':
        if request.form['submit_button'] == 'Return':
          return redirect('/')

        elif request.form['submit_button'] == 'View':
          return redirect('/view')

        elif request.form['submit_button'] == 'Edit':
            #Get all the details for what we want changed
            userDetails = request.form
            prevDetails = request.form
            product_id = userDetails['product_id']
            name = userDetails['name']
            price = userDetails['price']
            department = userDetails['department']
            description = userDetails['description']
            store_URL = userDetails['store_URL']

            #update query, used product id as primary key and overwrites the rest
            cur = mysql.connection.cursor()
            cur.execute("UPDATE inventory SET name = (%s), price = (%s),department = (%s),description = (%s), store_URL = (%s) WHERE product_id = (%s)",(name,price,department,description,store_URL, [product_id]))
            mysql.connection.commit()
            cur.close()
    return render_template('edit.html', prevDetails = prevDetails)

""" ---- Delete API ----- """
@app.route('/delete',methods=['GET', 'POST'])
def delete():
    if request.method == 'POST':
        if request.form['submit_button'] == 'Return':
          return redirect('/')

        if request.form['submit_button'] == 'Delete':
            userDetails = request.form
            product_id = userDetails['product_id']
            name = userDetails['name']
            cur = mysql.connection.cursor()
            #delete query, uses product id to identify unique item to delete
            cur.execute("DELETE From inventory WHERE product_id = (%s)", [product_id])
            mysql.connection.commit()
            cur.close()
        if request.form['submit_button'] == 'View':
          return redirect('/view')
          #render deleted product id and name on the delete page
    return render_template('delete.html',product_id = product_id, name = name)

""" ---- View API ----- """
@app.route('/view',methods=['GET', 'POST'])
def view():
    if request.method == 'POST':
        #return to main maenu
        if request.form['submit_button'] == 'Return':
          return redirect('/')

        #redirect to edit page
        elif request.form['submit_button'] == 'Edit':
          return redirect('/edit')

        #redirect to delete page
        elif request.form['submit_button'] == 'Delete':
          return redirect('/delete')

        #redirect to export function
        elif request.form['submit_button'] == 'Export':
          userDetails = request.form
          product_id = userDetails['product_id']
          export(product_id)

    cur = mysql.connection.cursor()
    cur.execute("SELECT * FROM inventory")
    userDetails = cur.fetchall()
    return render_template('view.html',userDetails=userDetails)

""" ---- export function ----- """
def export(product_id):
    # export to csv functionality
               header = ['Product ID', 'Name','Price', 'Department', 'Description','Store URL']

               #Had to use MySQLdb to connect instead of mysql.connection for it to work
               db = MySQLdb.connect("localhost","root","","shopify_inventory")
               cur = db.cursor()
               cur.execute("SELECT * FROM inventory WHERE product_id = (%s)", [product_id])

               result = cur.fetchall()
               c = csv.writer(open('product_data.csv', 'w',encoding='utf-8'))
               c.writerow(header)
               #writes information in the csv file
               for row in result:
                   print(product_id)
                   c.writerow(row)

if __name__ == '__main__':
    app.run(debug=True)


from django.shortcuts import render
from blog.models import ArticleModel
from django.core.paginator import Paginator
from django.db.models import Q
#from IpBlock.ip_blocking import BlockedIpMiddleware

def homePage(request):
    search = request.GET.get('search')
    articles = ArticleModel.objects.order_by('-id')

    if search:
        articles = articles.filter(
            Q(title__icontains=search) |
            Q(content__icontains=search)
        ).distinct()

    page = request.GET.get('page')
    paginator = Paginator(articles, 3)

    return render(request, 'pages/homepage.html', context={
        'articles': paginator.get_page(page)
    })


import logging

logger = logging.getLogger('upgrade')

def migrate(cr, installed_version):

	logger.info("Migrating Staff management to version %s", installed_version)

	cr.execute("UPDATE staff_scheduler s SET work_time = (Select (hour_to - hour_from) w from staff_scheduler t where s.id = t.id)")

	logger.info("Staff Management update... done!")


from flask import Flask, jsonify, request

app = Flask(__name__, template_folder='templates', static_folder='')
app.debug = True

@app.route("/api/getData", methods=["GET"])
def get_data():
	"""get请求"""
	return jsonify({"code": 200, "msg": "success", "data": request.args}), 200

@app.route("/api/post", methods=["POST"])
def post():
	"""post请求"""
	request_content_type = request.headers['Content-Type'].split(";")[0].strip()
	if request_content_type == 'application/json':
		return jsonify({"code": 200, "msg": "success", "data": request.get_json()}), 200
	return jsonify({"code": 200, "msg": "success"}), 200

if __name__ == '__main__':
	app.run()


#!/usr/bin/python3

""" Load typology questionnaires and parse them into a working format """

import pandas
from xlrd.biffh import XLRDError

class UnexpectedTypologyFormatError (ValueError):
    """Exception to be thrown when a questionnaire is misformatted"""
    pass

def parse_typology_questionnaire(filename):
    questionnaire = pandas.ExcelFile(filename)

    metadata_sheet_name = 'Metadata'
    if metadata_sheet_name in questionnaire.sheet_names:
        metadata_sheet = questionnaire.parse(metadata_sheet_name, header=None)
        try:
            metadata_blob = "; ".join(metadata_sheet[0])
        except KeyError:
            metadata_blob = ""
    else:
        metadata_blob = ""
    metadata = metadata_blob

    weird_rows = set()
    answers = pandas.DataFrame(columns = ["a_text", "a_notes" ,"a_reference"])
    features = pandas.DataFrame(columns = ["q_id: Reesink", "q_text", "q_note: Nijmegen Typological Survey"])
    data_sheet_name = 'Questionnaire'
    try:
        data_sheet = questionnaire.parse(
            data_sheet_name,
            skiprows=[0, 1, 2, 3, 4, 5, 7]).dropna('columns', 'all')
    except XLRDError:
        raise UnexpectedTypologyFormatError("Typology sheet did not have a 'Questionnaire' sheet")
    for i, row in data_sheet.iterrows():
        if pandas.isnull(row["q_text"]):
            weird_rows.add(i)
        if pandas.isnull(row["a_text"]):
            weird_rows.add(i)
        else:
            features = features.append(row[["q_id: Reesink", "q_text", "q_note: Nijmegen Topological Survey"]])
            answer = row[["a_text", "a_notes", "a_reference"]]
            answer.name = row["q_text"].lower()
            answers = answers.append(answer)
            assert(len(features) == len(answers))
    return metadata, features, answers

if __name__=='__main__':
    import sys
    metadata, features, answers = parse_questionnaire(sys.argv[1])
    print(metadata)
    print(features)
    print(answers.to_string())


import numpy as np
from keras.models import Sequential
from keras.layers import BatchNormalization, Conv1D, LSTM, Dense, Dropout
from keras.callbacks import *
from keras.utils import np_utils
def get_available_gpus():
    from tensorflow.python.client import device_lib
    local_device_protos = device_lib.list_local_devices()
    return [x.name for x in local_device_protos if x.device_type == 'GPU']

#load data
file_path = 'data\\GoT.txt'
book = open(file_path, encoding = "utf8").read().lower()

#covert unique chars to int
chars = sorted(list(set(book)))

char_to_int = dict((c, i) for i, c in enumerate(chars))
int_to_char = dict((i, c) for i, c in enumerate(chars))

char_int_dict = {}
for integer, character in enumerate(chars):
	char_int_dict[character] = integer

total_chars = len(book)
unique_chars = len(chars)

filter_len = 250 #100
dataX = []
dataY = []
for i in range(0, total_chars - filter_len):
    lineX = book[i: i + filter_len]
    lineY = book[i + filter_len]

	#char_int_list = []
	#for char in lineX:
	#	char_int_list.append(char_int_dict[char])

	#dataX.append(char_int_list)
	#dataY.append(char_int_dict[lineY])
    dataX.append([char_to_int[char] for char in lineX])
    dataY.append(char_to_int[lineY])
total_filters = len(dataX)

#prepare training data
X_train = np.reshape(dataX, (total_filters, filter_len, 1))
X_train = X_train / float(unique_chars)
y_train = np_utils.to_categorical(dataY)

#print(X_train.shape)
# (144225, 100, 1)
#print(y_train.shape)
#(144225, 45)

'''
LSTM RNN MODEL

Pre: Input into this model (train,valid) should have been preprocessed and
reshaped into the following data format for X: [samples, time steps, features]
Post: Generates a keras sequential model instance to generate text predictions.
Model consists of a LSTM (Long Short Term Memory] layers.
'''

#========================MODEL==================++

starting_neurons =  16 # starting filters

if len(get_available_gpus())>0:
    # https://twitter.com/fchollet/status/918170264608817152?lang=en
    from keras.layers import CuDNNLSTM as LSTM # this one is about 3x faster on GPU instances
stroke_read_model = Sequential()
stroke_read_model.add(LSTM(starting_neurons * 16,input_shape=(X_train.shape[1], X_train.shape[2]) ,return_sequences = True))
stroke_read_model.add(Dropout(0.2))
stroke_read_model.add(LSTM(starting_neurons * 16, return_sequences = False))
stroke_read_model.add(Dropout(0.2))
#stroke_read_model.add(Dense(starting_neurons * 32))
#stroke_read_model.add(Dropout(0.2))
stroke_read_model.add(Dense(y_train.shape[1], activation = 'softmax'))
stroke_read_model.compile(optimizer = 'adam', loss = 'categorical_crossentropy', metrics = ['categorical_accuracy'])

#=======================HyperParameters===========

weight_path="weights_got.best.hdf5"
checkpoint = ModelCheckpoint(weight_path, monitor='loss', verbose=1,
                             save_best_only=True, mode='min', save_weights_only = True)
reduceLROnPlat = ReduceLROnPlateau(monitor='loss', factor=0.8, patience=4,
                                   verbose=1, mode='auto', min_lr=0.0001)
early = EarlyStopping(monitor="val_loss", mode="min", patience=20)
callbacks_list = [checkpoint, reduceLROnPlat] #early,

#========================TRAINING=================
from IPython.display import clear_output
stroke_read_model.fit(X_train, y_train,
                      #validation_split = 0.2,
                      batch_size = 512,#128,
                      epochs = 100,
                      callbacks = callbacks_list,
                      verbose = 1)
clear_output()

#========================PREDICTIONS==============

'''
model.load_weights('weights.best.hdf5')
stroke_read_model.compile(optimizer = 'adam', loss = 'categorical_crossentropy')

# reverse mapping to convert integers back into characters
int_to_char = dict((i, c) for i, c in enumerate(chars))

# pick a random seed
start = numpy.random.randint(0, len(dataX)-1)
pattern = dataX[start]
print ("Seed:")
print "\"", ''.join([int_to_char[value] for value in pattern]), "\""
# generate characters
for i in range(1000):
	x = numpy.reshape(pattern, (1, len(pattern), 1))
	x = x / float(n_vocab)
	prediction = model.predict(x, verbose=0)
	index = numpy.argmax(prediction)
	result = int_to_char[index]
	seq_in = [int_to_char[value] for value in pattern]
	sys.stdout.write(result)
	pattern.append(index)
	pattern = pattern[1:len(pattern)]
print "Done."
'''


from __future__ import absolute_import, division, print_function
from six.moves import range
import math
import scitbx.math
from scitbx import lbfgs
from scitbx.array_family import flex

class fit_cdf(object):
  """
  =============================================================================
  Class fits a distribution according to its cumulative distribution function

  Arguments:
    x_data - measured property (list)
    y_data - fraction of points with measured property (cdf) (list)
    distribution - type of distribution to be fit

  Useful accessible attributes:
    self.x - the final parameters (list)
    self.distribution - the distribution being modeled (see notes)

  Notes:
    The available distributions are,
      gaussian
      rayleigh
      rice
    The distribution argument should not have quotes since it is used directly
    to access the class.
    The lbfgs method is used to minimize, f = (predicted - observed)^2
  -----------------------------------------------------------------------------
  """
  def __init__(self,x_data=None,y_data=None,distribution=None,
               max_iterations=50):
    # setup data
    assert(len(x_data) == len(y_data))
    self.n = len(x_data)
    self.x_data = flex.double(x_data)
    self.y_data = flex.double(y_data)
    if (flex.max(self.y_data) > 1.0):
      print("The cumulative distribution function (y_data) should only have values be between 0 and 1.")
      exit()

    # intialize distribution with guess
    self.distribution = distribution()
    self.distribution.estimate_parameters_from_cdf(x_data=self.x_data,
                                                   y_data=self.y_data)
    self.x = self.distribution.get_parameters()

    # optimize parameters
    self.minimizer = lbfgs.run(target_evaluator=self)

    # set optimized parameters
    self.distribution.set_parameters(p=self.x)

  def compute_functional_and_gradients(self):
    # caculate difference between predicted and observed values
    self.distribution.set_parameters(p=self.x)
    is_cpp_ = getattr(self.distribution,"interface","Python")=="C++"
    if is_cpp_:
      predicted = self.distribution.cdf(x=self.x_data)
    else:
      predicted = flex.double(self.n)
      for i in range(self.n):
        predicted[i] = self.distribution.cdf(x=self.x_data[i])
    difference = predicted - self.y_data

    # target function for minimization is sum of rmsd
    f = flex.sum(flex.sqrt(difference*difference))
    if is_cpp_:
      gradients = self.distribution.gradients(x=self.x_data, nparams=len(self.x), difference=difference)
      return f,gradients
    gradients = flex.double(len(self.x))
    for i in range(self.n):
      g_i = self.distribution.cdf_gradients(x=self.x_data[i])
      for j in range(len(self.x)):
        gradients[j] = gradients[j] + difference[i]*g_i[j]
    gradients = 2.0*gradients
    return f,gradients

# =============================================================================
class rayleigh(object):
  """
  =============================================================================
  Class models a 1-d Rayleigh distribution using one parameter, sigma.

              x                x^2
    pdf = --------- exp(- ------------)
           sigma^2         2 sigma^2

                        x^2
    cdf = 1 - exp(- -----------)
                     2 sigma^2

  The derivative of the cdf with respect to sigma is,

      d(cdf)          x^2               x^2             x
    ---------- = - --------- exp( - -----------) = - ------- pdf
     d(sigma)       sigma^3          2 sigma^2        sigma

  Methods:
    set_parameters
    get_parameters
    estimate_parameters_from_cdf
    pdf
    cdf
    d_cdf_d_sigma
    d_cdf_d_sigma_finite
    cdf_gradients
  -----------------------------------------------------------------------------
  """
  def __init__(self,mean=None,sigma=None):

    # parameter for Rayleigh distribution
    if (sigma is None):
      sigma = 1.0
    self.sigma = sigma

  def set_parameters(self,p=None):
    """
    Function sets the all the parameters
    """
    assert(len(p) == 1)
    self.sigma = float(p[0])

  def get_parameters(self):
    """
    Function returns all the parameters
    """
    return flex.double([self.sigma])

  def estimate_parameters_from_cdf(self,x_data=None,y_data=None):
    """
    Function estimates the parameter values based on the data (cdf)
    """
    # sigma is the mode of the distribution
    # approximate with the median (cdf = 0.5)
    midpoint = None
    for i in range(len(x_data)):
      if (y_data[i] > 0.5):
        midpoint = i
        break
    if (midpoint is None):
      midpoint = len(x_data) - 1
    self.sigma = x_data[midpoint]

  def pdf(self,x=None):
    """
    Function returns the probability density function at x
    """
    x_sigma = x/self.sigma
    f = (x_sigma/self.sigma)*math.exp(-0.5*x_sigma*x_sigma)
    return f

  def cdf(self,x=None):
    """
    Function returns the cumulative distribution function at x
    """
    x_sigma = x/self.sigma
    f = 1.0 - math.exp(-0.5*x_sigma*x_sigma)
    return f

  def inv_cdf(self,cdf=None):
    """
    Function returns the inverse cumulative distribution function at cdf
    """
    return math.sqrt(-2.*self.sigma*self.sigma*math.log(1.-cdf))

  def d_cdf_d_sigma(self,x=None):
    """
    Function returns the derivative of the cdf at x with respect to the
    standard deviation
    """
    df = -(x/self.sigma)*self.pdf(x=x)
    return df

  def d_cdf_d_sigma_finite(self,x=None,delta=0.00001):
    """
    Function returns the derivative of the cdf at x with respect to the
    standard deviation
    """
    sigma0 = self.sigma
    self.sigma = sigma0 + delta
    f1 = self.cdf(x=x)
    self.sigma = sigma0 - delta
    f2 = self.cdf(x=x)
    self.sigma = sigma0
    return (f1-f2)/(2.0*delta)

  def cdf_gradients(self,x=None):
    """
    Function returns a flex.double containing all derivatives
    """
    return flex.double([self.d_cdf_d_sigma(x)])


"""Utility functions for SLaQ"""
import numpy as np
from igraph import Graph
import scipy.sparse
from scipy.sparse.base import spmatrix

def laplacian(adjacency, normalized = True):
    """Computes the sparse Laplacian matrix given sparse adjacency matrix as input.

    Args:
        adjacency (spmatrix): Input adjacency matrix of a graph.
        normalized (bool): If True, return the normalized version of the Laplacian.

    Returns:
        spmatrix: Sparse Laplacian matrix of the graph.
    """
    degree = np.squeeze(np.asarray(adjacency.sum(axis=1)))
    if not normalized:
        return scipy.sparse.diags(degree) - adjacency
    with np.errstate(divide='ignore'): # Ignore the warning for divide by 0 case.
        degree = 1. / np.sqrt(degree)
    degree[degree == np.inf] = 0
    degree = scipy.sparse.diags(degree)
    return scipy.sparse.eye(adjacency.shape[0], dtype=np.float32) - degree @ adjacency @ degree

def get_adjacency(graph):
    vcount = graph.vcount()
    adjacency = scipy.sparse.dok_matrix((vcount, vcount), dtype=np.float32)
    for e in graph.get_edgelist():
        adjacency[e[0], e[1]] = 1
        adjacency[e[1], e[0]] = 1
    # Convert to the compressed sparse row format for efficiency.
    adjacency = adjacency.tocsr()
    adjacency.data = np.ones(adjacency.data.shape, dtype=np.float32) # Set all elements to one in case of duplicate rows.

    return adjacency


import colander
import deform.widget
from persistent.dict import PersistentDict
from zope.interface import implementer

from substanced.content import content
from substanced.schema import NameSchemaNode
from substanced.util import renamer, get_oid

from dace.descriptors import (
    SharedUniqueProperty, CompositeMultipleProperty,
    SharedMultipleProperty)
from dace.util import getSite, get_obj
from pontus.core import VisualisableElementSchema
from pontus.widget import (
    Select2Widget, AjaxSelect2Widget)
from pontus.schema import Schema
from pontus.file import ObjectData, File

from .interface import IComment
from novaideo.core import (
    Commentable, Emojiable, can_access, SignalableEntity,
    CorrelableEntity)
from novaideo import _, log
from novaideo.content import get_file_widget
from novaideo.utilities.util import (
    text_urls_format,
    get_emoji_form,
    get_files_data, connect, disconnect)
from novaideo.content.correlation import CorrelationType
from novaideo.widget import FilesWidget

@colander.deferred
def intention_choice(node, kw):
    root = getSite()
    values = [(str(i), i) for i in root.comment_intentions]
    values.insert(0, ('', _('- Select -')))
    return Select2Widget(
        values=values,
        item_css_class="comment-form-group comment-intention-form")

@colander.deferred
def relatedcontents_choice(node, kw):
    request = node.bindings['request']
    context = node.bindings['context']
    root = getSite()
    values = []
    if isinstance(context, Comment) and\
       context.related_correlation:
        values = [(get_oid(t), t.title) for
                  t in context.associated_contents]

    def title_getter(id):
        try:
            obj = get_obj(int(id), None)
            if obj:
                return obj.title
            else:
                return id
        except Exception as e:
            log.warning(e)
            return id

    ajax_url = request.resource_url(
        root, '@@novaideoapi',
        query={'op': 'find_correlable_entity'})
    return AjaxSelect2Widget(
        values=values,
        ajax_url=ajax_url,
        ajax_item_template="related_item_template",
        css_class="search-idea-form",
        title_getter=title_getter,
        multiple=True,
        page_limit=50,
        item_css_class="comment-form-group comment-related-form")

class RelatedContentsSchema(Schema):
    """Schema for associtation"""

    associated_contents = colander.SchemaNode(
        colander.Set(),
        widget=relatedcontents_choice,
        title=_('Associated contents'),
        description=_('Choose the contents to be associated'),
        missing=[],
        default=[],
        )

    associate = colander.SchemaNode(
        colander.Boolean(),
        widget=deform.widget.CheckboxWidget(
            item_css_class="hide-bloc"),
        label='',
        title='',
        default=False,
        missing=False
        )

@colander.deferred
def comment_textarea(node, kw):
    request = node.bindings['request']
    emoji_form = get_emoji_form(
        request, emoji_class='comment-form-group')
    return deform.widget.TextAreaWidget(
        rows=2, cols=60, item_css_class="comment-form-group comment-textarea",
        emoji_form=emoji_form,
        template='novaideo:views/templates/textarea_comment.pt',
        request=request)

def context_is_a_comment(context, request):
    return request.registry.content.istype(context, 'comment')

class CommentSchema(VisualisableElementSchema):
    """Schema for comment"""

    name = NameSchemaNode(
        editing=context_is_a_comment,
        )

    intention = colander.SchemaNode(
        colander.String(),
        widget=intention_choice,
        title=_('Intention'),
        description=_('Choose your intention'),
        default=_('Remark')
        )

    associated_contents = colander.SchemaNode(
        colander.Set(),
        widget=relatedcontents_choice,
        title=_('Associated contents'),
        description=_('Choose contents to associate'),
        missing=[],
        default=[],
        )

    files = colander.SchemaNode(
        colander.Sequence(),
        colander.SchemaNode(
            ObjectData(File),
            name=_("File"),
            widget=get_file_widget()
            ),
        widget=FilesWidget(
            add_subitem_text_template='',
            item_css_class="files-block comment-form-group comment-files"),
        missing=[],
        description=_('Add files to your comment'),
        title=_('Attached files'),
        )

    comment = colander.SchemaNode(
        colander.String(),
        validator=colander.Length(max=2000),
        widget=comment_textarea,
        title=_("Message")
        )

    anonymous = colander.SchemaNode(
        colander.Boolean(),
        widget=deform.widget.CheckboxWidget(
            item_css_class="comment-form-group comment-anonymous-form"),
        label=_('Remain anonymous'),
        description=_('Check this box if you want to remain anonymous.'),
        title='',
        missing=False,
        default=False
        )

@content(
    'comment',
    icon='glyphicon glyphicon-align-left',
    )
@implementer(IComment)
class Comment(Commentable, CorrelableEntity, Emojiable, SignalableEntity):
    """Comment class"""
    icon = 'icon ion-chatbubbles'
    templates = {'default': 'novaideo:views/templates/comment_result.pt'}
    name = renamer()
    author = SharedUniqueProperty('author')
    files = CompositeMultipleProperty('files')
    url_files = CompositeMultipleProperty('url_files')
    related_correlation = SharedUniqueProperty('related_correlation', 'targets')
    contextualized_correlations = SharedMultipleProperty(
        'contextualized_correlations', 'context')

    def __init__(self, **kwargs):
        super(Comment, self).__init__(**kwargs)
        self.set_data(kwargs)
        self.urls = PersistentDict({})
        self.edited = False
        self.pinned = False

    @property
    def channel(self):
        """Return the channel of th commented entity"""

        if not isinstance(self.__parent__, Comment):
            return self.__parent__
        else:
            return self.__parent__.channel

    @property
    def root(self):
        """Return the root comment"""

        if not isinstance(self.__parent__, Comment):
            return self
        else:
            return self.__parent__.root

    @property
    def comment_parent(self):
        """Return the root comment"""

        if isinstance(self.__parent__, Comment):
            return self.__parent__
        else:
            return None

    @property
    def subject(self):
        """Return the commented entity"""
        return self.channel.get_subject()

    @property
    def challenge(self):
        return getattr(self.subject, 'challenge', None)

    @property
    def relevant_data(self):
        return [getattr(self, 'comment', ''),
                getattr(self.author, 'title',
                        getattr(self.author, '__name__', ''))]

    @property
    def related_contents(self):
        subject = self.subject
        return [content[0] for content in self.contextualized_contents
                if content[0] is not subject]

    @property
    def associated_contents(self):
        subject = self.subject
        return [content[0] for content in self.contextualized_contents
                if content[0] is not subject and not getattr(content[1], 'tags', [])]

    def set_associated_contents(self, associated_contents, user):
        subject = self.subject
        current_associated_contents = self.associated_contents
        associated_contents_to_add = [i for i in associated_contents
                                      if i not in current_associated_contents]
        associated_contents_to_del = [i for i in current_associated_contents
                                      if i not in associated_contents and
                                      i not in associated_contents_to_add]
        correlations = connect(
            subject,
            associated_contents_to_add,
            {'comment': _('Add related contents'),
             'type': _('Edit the comment')},
            author=user,)
        for correlation in correlations:
            correlation.setproperty('context', self)

        disconnect(
            subject,
            associated_contents_to_del)

    def get_title(self):
        return self.subject.title

    def presentation_text(self, nb_characters=400):
        return getattr(self, 'comment', "")[:nb_characters]+'...'

    def get_discuss_url(self, request, user):
        subject = self.channel.get_subject(user)
        return request.resource_url(
            subject, "@@index") + '#comment-' + str(get_oid(self, 'None'))

    def get_url(self, request):
        return request.resource_url(
            request.root, "@@seecomment", query={'comment_id': get_oid(self)})

    def get_related_contents(self, user):
        return [r for r in self.related_contents if can_access(user, r)]

    def format(self, request, is_html=False):
        comment = getattr(self, 'comment', '')
        all_urls, url_files, text_urls, formatted_text = text_urls_format(
            comment, request, is_html)
        self.urls = PersistentDict(all_urls)
        self.setproperty('url_files', url_files)
        self.formatted_comment = formatted_text
        self.formatted_urls = text_urls

    def get_attached_files_data(self):
        return get_files_data(self.files)

    def can_add_reaction(self, user, process):
        return True


import lib
import logging
import sys

class LocalAccountReport(object):
    def __init__(self):
        self.config = lib.ConfigHelper()
        self.server = lib.ServerController()
        self.dict = lib.DictController(self.config)
        self.csv_writer = lib.CsvWriter(self.config)
        self.csv_columns = self.config.csv_columns

    def get_servers(self, args):
        return self.server.list_servers_under_group(args)

    def run(self, servers):
        for server in servers:
            accounts = self.server.list_local_accounts(server["id"])
            for account in accounts:
                data = self.dict.build_dict(account, server["id"])
                self.csv_writer.write(self.csv_columns.values(), data)
def main():
    sys.stdout.write("start retreiving server information")
    report = LocalAccountReport()

    args = report.config.argparse()

    servers = report.get_servers(args)

    report.run(servers)
    sys.stdout.write("finish writing out report")

if __name__ == "__main__":
    sys.stdout = lib.Logger(logging.info)
    sys.stderr = lib.Logger(logging.warning)
    main()


import scrapy
import logging
import re
from scrapy.selector import Selector
from urllib.parse import urljoin
from landchina.landchina_com.items import LandChinaItem, Payment
from sqlalchemy.orm import sessionmaker
from sqlalchemy.sql import exists
from landchina.landchina_com.models.models import Record, create_engine
from scrapy.utils.project import get_project_settings

class LandSpider(scrapy.Spider):
    name = "landchina_com"
    root_url = "http://www.landchina.com/default.aspx?tabid=263"
    encoding = "gbk"
    start_date = "2018-6-20"
    end_date = "2018-6-25"
    begin_page = "1"
    postData = {'TAB_QueryConditionItem': '9f2c3acd-0256-4da2-a659-6949c4671a2a',
                'TAB_QuerySubmitConditionData': '9f2c3acd-0256-4da2-a659-6949c4671a2a:{}~{}'.format(
                    start_date, end_date),
                'TAB_QuerySortItemList': '282:False',
                'TAB_QuerySubmitPagerData': begin_page,
                'hidComName': 'default',
                '__VIEWSTATE': "",
                '__EVENTVALIDATION': ""}
    engine = create_engine(get_project_settings().get("CONNECTION_STRING"))
    session = sessionmaker(bind=engine)()

    def start_requests(self):
        self.log("begin login for {} - {}".format(self.start_date, self.end_date), logging.INFO)
        return [scrapy.Request(url=self.root_url, callback=self.parse_login, encoding=self.encoding)]
        # return [scrapy.Request(url='http://www.landchina.com/default.aspx?tabid=386&comname=default&wmguid=75c72564-ffd9-426a-954b-8ac2df0903b7&recorderguid=3a4498fe-ba10-4080-9cfd-e78f16b03db1', callback=self.parse_detail, encoding=self.encoding)]

    def parse_login(self, response):
        self.log("process login ...", logging.INFO)
        resp = Selector(response)
        self.retrieve_state(resp)

        self.log("begin page {} for {} - {}".format(self.postData["TAB_QuerySubmitPagerData"], self.start_date,
                                                    self.end_date), logging.INFO)
        return scrapy.FormRequest(url=self.root_url, callback=self.parse_list, formdata=self.postData,
                                  meta=self.postData, encoding=self.encoding)

    def parse_list(self, response):
        self.log("process list page {}".format(response.url), logging.INFO)
        resp = Selector(response)
        table = resp.xpath('//table[@id="TAB_contentTable"]')[0]

        # push detail page links to scrapy
        detail_links = table.xpath('//a[contains(@href, "tabid=386")]/@href').extract()
        for link in detail_links:
            matchObj = re.match(r'.*recorderguid=(.*)', link, re.M | re.I)
            if matchObj and self.record_exist(matchObj[1]):
                continue
            yield scrapy.Request(url=urljoin(self.root_url, link), callback=self.parse_detail, meta=response.meta,
                                 encoding=self.encoding)

        # push page links to scrapy
        if response.meta["TAB_QuerySubmitPagerData"] == self.begin_page:
            self.retrieve_state(resp)
            totalPage=int(self.begin_page) + 1
            try:
                totalPage = table.xpath("//a[contains(., '尾页')]").re_first(r"QueryAction.GoPage\('TAB',(\d*)").strip()
            except:
                pass
            for pageNo in range(int(self.begin_page), int(totalPage) + 1):
                self.log("begin page {} for {} - {}".format(pageNo, self.start_date, self.end_date), logging.INFO)
                self.postData["TAB_QuerySubmitPagerData"] = str(pageNo)
                yield scrapy.FormRequest(url=self.root_url, callback=self.parse_list, formdata=self.postData,
                                         meta=self.postData, encoding=self.encoding)

    def parse_detail(self, response):
        self.log("process detail page {}, {}".format(response.meta["TAB_QuerySubmitPagerData"], response.url),
                 logging.INFO)
        resp = Selector(response)
        childTrs = resp.xpath('//table[@class="theme"]/tbody/tr')
        data = LandChinaItem()
        data["url"] = response.url
        data["guid"] = resp.xpath('//form/@action').re_first(r'recorderguid=(.*)')
        data["district"] = childTrs[2].xpath('td[2]/span/text()').extract_first()
        data["regulationNo"] = childTrs[2].xpath('td[4]/span/text()').extract_first()
        data["name"] = childTrs[3].xpath('td[2]/span/text()').extract_first()
        data["location"] = childTrs[4].xpath('td[2]/span/text()').extract_first()
        data["area"] = childTrs[5].xpath('td[2]/span/text()').extract_first()
        are2 = childTrs[5].xpath('td[4]/span/text()').extract()
        try:
            if are2 == "" or not are2:
                data["landSource"] = "新增建设用地(来自存量库)"
            elif float(data["area"]) == float(are2):
                data["landSource"] = "现有建设用地"
            elif float(are2) == 0:
                data["landSource"] = "新增建设用地"
            else:
                data["landSource"] = "新增建设用地(来自存量库)"
        except:
            data["landSource"] = "新增建设用地(来自存量库)"
        data["usage"] = childTrs[6].xpath('td[2]/span/text()').extract_first()
        data["wayOfSupply"] = childTrs[6].xpath('td[4]/span/text()').extract_first()
        data["tenureOfUse"] = childTrs[7].xpath('td[2]/span/text()').extract_first()
        data["industry"] = childTrs[7].xpath('td[4]/span/text()').extract_first()
        data["landLevel"] = childTrs[8].xpath('td[2]/span/text()').extract_first()
        data["price"] = childTrs[8].xpath('td[4]/span/text()').extract_first()
        data["owner"] = childTrs[10].xpath('td[2]/span/text()').extract_first()
        plotRatio = childTrs[12].xpath('td/table//tr[2]/td')
        data["plotRatioDownLimit"] = plotRatio[1].xpath('span/text()').extract_first()
        data["plotRatioUpperLimit"] = plotRatio[3].xpath('span/text()').extract_first()
        data["dateOfDeliveryAgreed"] = childTrs[12].xpath('td[4]/span/text()').extract_first()
        data["dateOfConstructionAgreed"] = childTrs[13].xpath('td[2]/span/text()').extract_first()
        data["dateOfCompletionAgreed"] = childTrs[13].xpath('td[4]/span/text()').extract_first()
        data["dateOfConstructionActual"] = childTrs[14].xpath('td[2]/span/text()').extract_first()
        data["dateOfCompletionActual"] = childTrs[14].xpath('td[4]/span/text()').extract_first()
        data["approvedBy"] = childTrs[15].xpath('td[2]/span/text()').extract_first()
        if data["approvedBy"] is not None:
            if "人民政府" not in data["approvedBy"]:
                data["approvedBy"] += "人民政府"
        data["dateOfSigning"] = childTrs[15].xpath('td[4]/span/text()').extract_first()
        paymentsTrs = childTrs[9].xpath('td[2]//tr[contains(@kvalue, "-")]')
        payments = []
        for paymentsTr in paymentsTrs:
            payment = Payment()
            payment["guid"] = paymentsTr.xpath('@kvalue').extract_first()
            payment["date"] = paymentsTr.xpath('td[2]/span/text()').extract_first()
            payment["amount"] = paymentsTr.xpath('td[3]/span/text()').extract_first()
            payment["comment"] = paymentsTr.xpath('td[4]/span/text()').extract_first()
            payments.append(dict(payment))
        data["payments"] = payments
        return data

    def parse(self, response):
        self.log("process parse func {}".format(response.url), logging.INFO)
        pass

    def retrieve_state(self, resp):
        vs = resp.xpath('//input[@id="__VIEWSTATE"]/@value').extract_first().strip()
        ev = resp.xpath('//input[@id="__EVENTVALIDATION"]/@value').extract_first().strip()
        self.postData['__VIEWSTATE'] = vs
        self.postData['__EVENTVALIDATION'] = ev

    def record_exist(self, guid):
        return self.session.query(exists().where(Record.guid == guid)).scalar()


################################################
### SVM best case's recall & fall-out graph  ###
###            SVM best case's AUC           ###
###    SVM best case's decision boundary     ###
################################################

import matplotlib.pyplot as plt
import matplotlib as mpl

from sklearn import datasets
from sklearn.externals.six.moves import xrange

from sklearn import svm
from sklearn.svm import SVC
import numpy as np
import time

from sklearn.metrics import accuracy_score
from sklearn import metrics
from sklearn.metrics import confusion_matrix
import seaborn as sns
import mglearn
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.svm import LinearSVC
from sklearn.metrics import classification_report
from sklearn.metrics import log_loss
from sklearn.metrics import roc_curve

from sklearn.metrics import auc
from scipy import interp
from itertools import cycle

def plot_svc_decision_function(model, ax=None, plot_support=True):
    """Plot the decision function for a 2D SVC"""
    if ax is None:
        ax = plt.gca()
    xlim = ax.get_xlim()
    ylim = ax.get_ylim()

    # create grid to evaluate model
    x = np.linspace(xlim[0], xlim[1], 30)
    y = np.linspace(ylim[0], ylim[1], 30)
    Y, X = np.meshgrid(y, x)
    xy = np.vstack([X.ravel(), Y.ravel()]).T
    P = model.decision_function(xy).reshape(X.shape)

    # plot decision boundary and margins
    ax.contour(X, Y, P, colors='k',
               levels=[-1, 0, 1], alpha=0.5,
               linestyles=['--', '-', '--'])

    # plot support vectors
    if plot_support:
        ax.scatter(model.support_vectors_[:, 0],
                   model.support_vectors_[:, 1],
                   s=300, linewidth=1, facecolors='none')
    ax.set_xlim(xlim)
    ax.set_ylim(ylim)

## Choose only one of the three flags.
recall_fallout = False
roc_curve_flag = False
svm_show = True

problem = 0
# problem = 1

if problem == 0:
    train_input = np.loadtxt('./data/p1_train_input.txt')
    train_target = np.loadtxt('./data/p1_train_target.txt')
    test_input = np.loadtxt('./data/p1_test_input.txt')
    test_target = np.loadtxt('./data/p1_test_target.txt')
else:
    train_input = np.loadtxt('./data/p2_train_input.txt')
    train_target = np.loadtxt('./data/p2_train_target.txt')
    test_input = np.loadtxt('./data/p2_test_input.txt')
    test_target = np.loadtxt('./data/p2_test_target.txt')

xx, yy = np.meshgrid(np.linspace(-3, 3, 500),
                     np.linspace(-3, 3, 500))

clf = SVC(kernel='rbf', C=10, gamma=10, random_state=100)
clf.fit(train_input, train_target)

y_pred = clf.predict(test_input)

cm = confusion_matrix(test_target, y_pred)
print("confusion_matrix")
print(cm)

print("classification_report")
print(classification_report(test_target, y_pred, target_names=['class 0', 'class 1']))

recall = cm[0][0] / (cm[0][0] + cm[0][1])
fallout = cm[1][0] / (cm[1][0] + cm[1][1])
print("recall = {:.5f}".format(recall))
print("fallout = {:.5f}".format(fallout))

fpr, tpr, thresholds = roc_curve(test_target, y_pred)
_fpr = 1-fpr[np.argmax(np.abs(tpr - fpr))]
_tpr = tpr[np.argmax(np.abs(tpr - fpr))]
print("Specificity = {:.5f}".format(_fpr))
print("Sensitivity = {:.5f}".format(_tpr))

_auc = auc(fpr, tpr)
print("AUC = {:.5f}".format(_auc))

########################################
############ Draw ROC curve ############
########################################
lw = 2
n_classes = 2

_test_target = []
for tar in test_target:
    if tar == 0.0:
        _test_target.append([1, 0])
    else:
        _test_target.append([0, 1])
_test_target = np.array(_test_target)

_y_pred = []
for pre in y_pred:
    if pre == 0.0:
        _y_pred.append([1, 0])
    else:
        _y_pred.append([0, 1])
_y_pred = np.array(_y_pred)

if recall_fallout == True:
    plt.plot(fpr, tpr, 'o-', label="Logistic Regression")
    plt.plot([0, 1], [0, 1], 'k--', label="random guess")
    plt.plot([fallout], [recall], 'ro', ms=10)
    plt.xlabel('Fall-Out')
    plt.ylabel('Recall')
    plt.show()

if roc_curve_flag == True:
    # Compute ROC curve and ROC area for each class
    fpr = dict()
    tpr = dict()
    roc_auc = dict()
    for i in range(n_classes):
        fpr[i], tpr[i], _ = roc_curve(_test_target[:, i], _y_pred[:, i])
        roc_auc[i] = auc(fpr[i], tpr[i])

    # Compute micro-average ROC curve and ROC area
    fpr["micro"], tpr["micro"], _ = roc_curve(_test_target.ravel(), _y_pred.ravel())
    roc_auc["micro"] = auc(fpr["micro"], tpr["micro"])

    # Compute macro-average ROC curve and ROC area

    # First aggregate all false positive rates
    all_fpr = np.unique(np.concatenate([fpr[i] for i in range(n_classes)]))

    # Then interpolate all ROC curves at this points
    mean_tpr = np.zeros_like(all_fpr)
    for i in range(n_classes):
        mean_tpr += interp(all_fpr, fpr[i], tpr[i])

    # Finally average it and compute AUC
    mean_tpr /= n_classes

    fpr["macro"] = all_fpr
    tpr["macro"] = mean_tpr
    roc_auc["macro"] = auc(fpr["macro"], tpr["macro"])
    ## Plot all ROC curves
    plt.figure(1)
    plt.plot(fpr["micro"], tpr["micro"],
            label='micro-average ROC curve (area = {0:0.5f})'
                ''.format(roc_auc["micro"]),
            color='deeppink', linestyle=':', linewidth=4)

    plt.plot(fpr["macro"], tpr["macro"],
            label='macro-average ROC curve (area = {0:0.5f})'
                ''.format(roc_auc["macro"]),
            color='navy', linestyle=':', linewidth=4)

    colors = cycle(['aqua', 'darkorange', 'cornflowerblue'])
    for i, color in zip(range(n_classes), colors):
        plt.plot(fpr[i], tpr[i], color=color, lw=lw,
                label='ROC curve of class {0} (area = {1:0.5f})'
                ''.format(i, roc_auc[i]))

    plt.plot([0, 1], [0, 1], 'k--', lw=lw)
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('ROC curve')
    plt.legend(loc="lower right")
    plt.show()

if svm_show == True:
    plt.scatter(train_input[:, 0], train_input[:, 1], c=train_target, s=50, cmap='autumn')
    plot_svc_decision_function(clf)
    plt.scatter(clf.support_vectors_[:, 0], clf.support_vectors_[:, 1],
                s=300, lw=1, facecolors='none')
    plt.show()


import pytest

from alarm_backends.core.cache.strategy import StrategyCacheManager
from constants.data_source import DataSourceLabel

pytestmark = pytest.mark.django_db

item = {
    "query_configs": [
        {
            "agg_method": "AVG",
            "agg_dimension": ["ip", "bk_cloud_id"],
            "agg_condition": [
                {"value": "0", "method": "eq", "key": "bk_cloud_id"},
                {"value": "10.0.1.11", "key": "bk_target_ip", "condition": "or", "method": "eq"},
            ],
            "agg_interval": 60,
            "metric_field": "usage",
            "result_table_id": "system.cpu_summary",
            "unit": "%",
            "metric_id": "system.cpu_summary.usage",
            "data_source_label": "bk_monitor",
            "data_type_label": "time_series",
        }
    ],
    "algorithms": [
        {
            "config": [{"threshold": 90, "method": "gte"}],
            "level": 1,
            "type": "Threshold",
        }
    ],
    "id": 6,
    "no_data_config": {"is_enabled": False, "continuous": 5},
    "name": "usage",
}

class TestStrategyGroup(object):
    def test_query_config_md5(self):
        query_md5 = StrategyCacheManager.get_query_md5(2, item)
        assert query_md5 == "4ca8defa2aed29a8371e665b610a2044"
        item["query_configs"][0]["agg_condition"] = [
            {"value": "10.0.1.11", "key": "bk_target_ip", "method": "eq"},
            {"value": "0", "method": "eq", "key": "bk_cloud_id", "condition": "or"},
        ]
        query_md5 = StrategyCacheManager.get_query_md5(2, item)
        assert query_md5 == "6e2170c2323fb98bdb9c31f6416cfd9e"

        item["query_configs"][0]["agg_condition"] = []
        empty_condition_query_md5 = StrategyCacheManager.get_query_md5(2, item)

        # 条件包含正则等高级方法时的查询md5
        item["query_configs"][0]["agg_condition"] = [{"value": "10.0.1.11", "key": "bk_target_ip", "method": "reg"}]
        query_md5 = StrategyCacheManager.get_query_md5(2, item)
        assert empty_condition_query_md5 != query_md5

        item["query_configs"][0]["data_source_label"] = DataSourceLabel.BK_DATA

        item["query_configs"][0]["agg_condition"] = []
        empty_condition_query_md5 = StrategyCacheManager.get_query_md5(2, item)

        item["query_configs"][0]["agg_condition"] = [{"value": "10.0.1.11", "key": "bk_target_ip", "method": "reg"}]
        query_md5 = StrategyCacheManager.get_query_md5(2, item)
        assert empty_condition_query_md5 == query_md5

    def test_cache(self):
        pass


from django.urls import path
from .views import *

app_name = 'accounts_api'

urlpatterns = [
    # path('change-password/<int:id>/', ChangePasswordView.as_view(), name='change-password'),
    path('add-to-wishlist/', AddToWishlist.as_view(), name='add-to-wishlist'),
    path('remove-from-wishlist/', RemoveFromWishlist.as_view(), name='remove-from-wishlist'),
    path('add-to-cart/', AddToCart.as_view(), name='add-to-cart'),
    path('remove-from-cart/', RemoveFromCart.as_view(), name='remove-from-cart'),
]


import sys

from queue import Queue
from PIL import Image
from convertImage import *

import pylab
import matplotlib.pyplot as plt
import matplotlib.pyplot as mplt
import numpy as np

#almostEqual for pixels
def almostEqual(x,y,epsilon=3):
    return abs(x[0]-y[0])<epsilon and abs(x[1]-y[1])<epsilon

##Input Start/Out Mode
#returns (start,end)
def readStartEnd(imageName):
    x1 = mplt.imread(imageName)
    fig1 = pylab.figure(1, figsize=(11,9))
    ax1 = fig1.add_subplot(1,1,1)
    ax1.imshow(x1)
    ax1.axis('image')
    ax1.axis('off')
    print("Click start and end coords")
    x = fig1.ginput(2)
    print("click the top left and bottom right corner")
    y = fig1.ginput(2)
    XBounds = (y[0][0],y[1][0])
    YBounds = (y[0][1],y[1][1])
    mplt.close(fig1)

    return(x[0],x[1],XBounds,YBounds)

#BFS general structure is taken from
#http://stackoverflow.com/questions/12995434/representing-and-solving-a-maze-given-an-image
#Modified from...

def BFS(start, end, pixels, XBounds, YBounds):
    lowXBound,highXBound = XBounds
    lowYBound,highYBound = YBounds

    def getAdjacent(n,step = 1):
        x,y = n
        return [(x-step,y),(x,y-step),(x+step,y),(x,y+step)] #ENWS

    def isLegal(pixels,x,y):
        nonlocal lowXBound,highXBound,lowYBound,highYBound
        R,B,G = pixels[x,y]
        t = 200
        if ((x < lowXBound) or (x > highXBound) or
        (y < lowYBound) or (y > highYBound) or
        R<t):
            return False # off-board!
        else: return True

    queue = Queue()
    queue.put([start]) # Wrapping the start tuple in a list

    while not queue.empty():

        path = queue.get()
        pixel = path[-1]
        #print(pixel,end = '  ')

        #if pixel == end:
        if almostEqual(pixel,end):
            return path

        #loop through all directions by factor of step
        for adjacent in getAdjacent(pixel,step=2):
            x,y = adjacent
            if isLegal(pixels,x,y):
                pixels[x,y] = (127,127,127) # see note
                new_path = list(path)
                new_path.append(adjacent)
                queue.put(new_path)
    print ("Queue has been exhausted. No answer was found.")

if __name__ == '__main__':

    mazeFile = "maze4.jpg"

    start,end,XBounds,YBounds = readStartEnd(mazeFile)

    base_img = Image.open(mazeFile)
    base_pixels = base_img.load()

    convertFromRGBToBW2(base_pixels,base_img)

    path = BFS(start, end, base_pixels, XBounds, YBounds)

    path_img = Image.open(mazeFile)
    path_pixels = path_img.load()

    for position in path:
        x,y = position
        path_pixels[x,y] = (255,0,0) # red

    path_img.show()