# Copyright (c) 2011 X.commerce, a business unit of eBay Inc.
# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
# Copyright 2013 Red Hat, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

# Interactive shell based on Django:
#
# Copyright (c) 2005, the Lawrence Journal-World
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#     1. Redistributions of source code must retain the above copyright notice,
#        this list of conditions and the following disclaimer.
#
#     2. Redistributions in binary form must reproduce the above copyright
#        notice, this list of conditions and the following disclaimer in the
#        documentation and/or other materials provided with the distribution.
#
#     3. Neither the name of Django nor the names of its contributors may be
#        used to endorse or promote products derived from this software without
#        specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""
  CLI interface for fs_gateway management.
"""

from __future__ import print_function

import argparse
import os
import six
import sys

from oslo_config import cfg
from oslo_log import log as logging

from conveyor import config
from conveyor import context
from conveyor import db
from conveyor import rpc
from conveyor import version

from conveyor.common import cliutils
from conveyor.db import api as db_api
from conveyor.db.sqlalchemy import migration
from conveyor.i18n import _

CONF = cfg.CONF

# Decorators for actions
def args(*args, **kwargs):
    def _decorator(func):
        func.__dict__.setdefault('args', []).insert(0, (args, kwargs))
        return func
    return _decorator

class ShellCommands(object):
    def bpython(self):
        """Runs a bpython shell.

        Falls back to Ipython/python shell if unavailable
        """
        self.run('bpython')

    def ipython(self):
        """Runs an Ipython shell.

        Falls back to Python shell if unavailable
        """
        self.run('ipython')

    def python(self):
        """Runs a python shell.

        Falls back to Python shell if unavailable
        """
        self.run('python')

    @args('--shell', metavar='<bpython|ipython|python >',
          help='Python shell')
    def run(self, shell=None):
        """Runs a Python interactive interpreter."""
        if not shell:
            shell = 'bpython'

        if shell == 'bpython':
            try:
                import bpython
                bpython.embed()
            except ImportError:
                shell = 'ipython'
        if shell == 'ipython':
            try:
                import IPython
                # Explicitly pass an empty list as arguments, because
                # otherwise IPython would use sys.argv from this script.
                shell = IPython.Shell.IPShell(argv=[])
                shell.mainloop()
            except ImportError:
                shell = 'python'

        if shell == 'python':
            import code
            try:
                # Try activating rlcompleter, because it's handy.
                import readline
            except ImportError:
                pass
            else:
                # We don't have to wrap the following import in a 'try',
                # because we already know 'readline' was imported successfully.
                readline.parse_and_bind("tab:complete")
            code.interact()

    @args('--path', metavar='<path>', help='Script path')
    def script(self, path):
        """Runs the script from the specified path with flags set properly.

        arguments: path
        """
        exec(compile(open(path).read(), path, 'exec'), locals(), globals())

def _db_error(caught_exception):
    print(caught_exception)
    print(_("The above error may show that the database has not "
            "been created.\nPlease create a database using "
            "'fs_gateway-manage db sync' before running this command."))
    exit(1)

class DbCommands(object):
    """Class for managing the database."""

    def __init__(self):
        pass

    @args('--version', metavar='<version>', help='Database version')
    def sync(self, version=None):
        """Sync the database up to the most recent version."""
        return migration.db_sync(db_api.get_engine(), version)

    def version(self):
        """Print the current database version."""
        print(migration.db_version())

    @args('--max_rows', metavar='<number>',
          help='Maximum number of deleted rows to archive')
    def archive_deleted_rows(self, max_rows):
        """Move up to max_rows deleted rows from production tables to shadow
        tables.
        """
        if max_rows is not None:
            max_rows = int(max_rows)
            if max_rows < 0:
                print(_("Must supply a positive value for max_rows"))
                return(1)
        admin_context = context.get_admin_context()
        db.archive_deleted_rows(admin_context, max_rows)

class GetLogCommands(object):
    """Get logging information."""

    def errors(self):
        """Get all of the errors from the log files."""
        error_found = 0
        if CONF.log_dir:
            logs = [x for x in os.listdir(CONF.log_dir) if x.endswith('.log')]
            for file in logs:
                log_file = os.path.join(CONF.log_dir, file)
                lines = [line.strip() for line in open(log_file, "r")]
                lines.reverse()
                print_name = 0
                for index, line in enumerate(lines):
                    if line.find(" ERROR ") > 0:
                        error_found += 1
                        if print_name == 0:
                            print(log_file + ":-")
                            print_name = 1
                        linenum = len(lines) - index
                        print((_('Line %(linenum)d : %(line)s') %
                               {'linenum': linenum, 'line': line}))
        if error_found == 0:
            print(_('No errors in logfiles!'))

    @args('--num_entries', metavar='<number of entries>',
          help='number of entries(default: 10)')
    def syslog(self, num_entries=10):
        """Get <num_entries> of the conveyor syslog events."""
        entries = int(num_entries)
        count = 0
        log_file = ''
        if os.path.exists('/var/log/syslog'):
            log_file = '/var/log/syslog'
        elif os.path.exists('/var/log/messages'):
            log_file = '/var/log/messages'
        else:
            print(_('Unable to find system log file!'))
            return(1)
        lines = [line.strip() for line in open(log_file, "r")]
        lines.reverse()
        print(_('Last %s fs_gateway syslog entries:-') % (entries))
        for line in lines:
            if line.find("fs_gateway") > 0:
                count += 1
                print("%s" % (line))
            if count == entries:
                break

        if count == 0:
            print(_('No fs_gateway entries in syslog!'))

CATEGORIES = {
    'db': DbCommands,
    'logs': GetLogCommands,
    'shell': ShellCommands,
}

def methods_of(obj):
    """Get all callable methods of an object that don't start with underscore

    returns a list of tuples of the form (method_name, method)
    """
    result = []
    for i in dir(obj):
        if callable(getattr(obj, i)) and not i.startswith('_'):
            result.append((i, getattr(obj, i)))
    return result

def add_command_parsers(subparsers):
    parser = subparsers.add_parser('version')

    parser = subparsers.add_parser('bash-completion')
    parser.add_argument('query_category', nargs='?')

    for category in CATEGORIES:
        command_object = CATEGORIES[category]()

        desc = getattr(command_object, 'description', None)
        parser = subparsers.add_parser(category, description=desc)
        parser.set_defaults(command_object=command_object)

        category_subparsers = parser.add_subparsers(dest='action')

        for (action, action_fn) in methods_of(command_object):
            parser = category_subparsers.add_parser(action, description=desc)

            action_kwargs = []
            for args, kwargs in getattr(action_fn, 'args', []):
                # FIXME(markmc): hack to assume dest is the arg name without
                # the leading hyphens if no dest is supplied
                kwargs.setdefault('dest', args[0][2:])
                if kwargs['dest'].startswith('action_kwarg_'):
                    action_kwargs.append(
                            kwargs['dest'][len('action_kwarg_'):])
                else:
                    action_kwargs.append(kwargs['dest'])
                    kwargs['dest'] = 'action_kwarg_' + kwargs['dest']

                parser.add_argument(*args, **kwargs)

            parser.set_defaults(action_fn=action_fn)
            parser.set_defaults(action_kwargs=action_kwargs)

            parser.add_argument('action_args', nargs='*',
                                help=argparse.SUPPRESS)

category_opt = cfg.SubCommandOpt('category',
                                 title='Command categories',
                                 help='Available categories',
                                 handler=add_command_parsers)

def main():
    """Parse options and call the appropriate class/method."""
    CONF.register_cli_opt(category_opt)
    try:
        config.parse_args(sys.argv)
        logging.setup(CONF, "conveyor")
    except cfg.ConfigFilesNotFoundError:
        cfgfile = CONF.config_file[-1] if CONF.config_file else None
        if cfgfile and not os.access(cfgfile, os.R_OK):
            st = os.stat(cfgfile)
            print(_("Could not read %s. Re-running with sudo") % cfgfile)
            try:
                os.execvp('sudo', ['sudo', '-u', '#%s' % st.st_uid] + sys.argv)
            except Exception:
                print(_('sudo failed, continuing as if nothing happened'))

        print(_('Please re-run gw-manage as root.'))
        return(2)

    if CONF.category.name == "version":
        print(version.version_string_with_package())
        return(0)

    if CONF.category.name == "bash-completion":
        if not CONF.category.query_category:
            print(" ".join(CATEGORIES.keys()))
        elif CONF.category.query_category in CATEGORIES:
            fn = CATEGORIES[CONF.category.query_category]
            command_object = fn()
            actions = methods_of(command_object)
            print(" ".join([k for (k, v) in actions]))
        return(0)

    fn = CONF.category.action_fn
    fn_args = [arg.decode('utf-8') for arg in CONF.category.action_args]
    fn_kwargs = {}
    for k in CONF.category.action_kwargs:
        v = getattr(CONF.category, 'action_kwarg_' + k)
        if v is None:
            continue
        if isinstance(v, six.string_types):
            v = v.decode('utf-8')
        fn_kwargs[k] = v

    # call the action with the remaining arguments
    # check arguments
    try:
        cliutils.validate_args(fn, *fn_args, **fn_kwargs)
    except cliutils.MissingArgs as e:
        # NOTE(mikal): this isn't the most helpful error message ever. It is
        # long, and tells you a lot of things you probably don't want to know
        # if you just got a single arg wrong.
        print(fn.__doc__)
        CONF.print_help()
        print(e)
        return(1)
    try:
        ret = fn(*fn_args, **fn_kwargs)
        rpc.cleanup()
        return(ret)
    except Exception:
        print(_("Command failed, please check log for more info"))
        raise


# Definition for singly-linked list.
# class ListNode(object):
#     def __init__(self, x):
#         self.val = x
#         self.next = None

class Solution(object):
    def middleNode(self, head):
        """
        :type head: ListNode
        :rtype: ListNode
        """
        if head == None:
            return None
        runner = head
        length = 0
        while runner:
            length += 1
            runner = runner.next
        for _ in range(length/2):
            head = head.next
        return head


"""
Prediction script for trained tensorflow PB model.
"""

import os
import time
import glob
import argparse
import numpy as np
import tensorflow as tf
import cv2

from PIL import Image

## Comment out next line to use GPU
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
## Comment out to set verbose to true
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"

def load_label_map(label_map_path):
    """
    Reads label map in the format of .pbtxt and parse into dictionary

    Args:
      label_map_path: the file path to the label_map

    Returns:
      dictionary with the format of {label_index: {'id': label_index, 'name': label_name}}
    """
    label_map = {}

    with open(label_map_path, "r") as label_file:
        for line in label_file:
            if "id" in line:
                label_index = int(line.split(":")[-1])
                label_name = next(label_file).split(":")[-1].strip().strip("'")
                label_map[label_index] = {"id": label_index, "name": label_name}

    return label_map

def load_image_into_numpy_array(path, height, width):
    """
    Load an image from file into a numpy array.

    Args:
      path: the file path to the image
      height: height of image
      width: width of image

    Returns:
      uint8 numpy array with shape (img_height, img_width, 3), (original_height, original_width)
    """
    image = Image.open(path).convert("RGB")
    image_shape = np.asarray(image).shape

    image_resized = image.resize((height, width))
    return np.array(image_resized), (image_shape[0], image_shape[1])

def args_parser():
    parser = argparse.ArgumentParser(
        description="Datature Open Source Prediction Script"
    )
    parser.add_argument(
        "--input", help="Path to folder that contains input images", required=True
    )
    parser.add_argument(
        "--output", help="Path to folder to store predicted images", required=True
    )
    parser.add_argument(
        "--size", help="Size of image to load into model", default="320x320"
    )
    parser.add_argument(
        "--threshold", help="Prediction confidence threshold", default=0.7
    )
    parser.add_argument(
        "--model", help="Path to tensorflow pb model", default="./saved_model"
    )
    parser.add_argument(
        "--label", help="Path to tensorflow label map", default="./label_map.pbtxt"
    )
    return parser.parse_args()

def main():
    ## Load argument variables
    args = args_parser()

    if os.path.exists(args.input) is False:
        raise Exception("Input Folder Path Do Not Exists")

    if os.path.exists(args.output) is False:
        raise Exception("Output Folder Path Do Not Exists")

    if os.path.exists(args.model) is False:
        raise Exception("Model Folder Do Not Exists")

    category_index = load_label_map(args.label)

    ## Load color map
    color_map = {}
    for each_class in range(len(category_index)):
        color_map[each_class] = [int(i) for i in np.random.choice(range(256), size=3)]

    ## Load model
    print("Loading model...")
    start_time = time.time()
    trained_model = tf.saved_model.load(args.model)
    print("Model loaded, took {} seconds...".format(time.time() - start_time))

    ## Run prediction on each image
    for each_image in glob.glob(os.path.join(args.input, "*")):
        print("Predicting for {}...".format(each_image))

        height, width = args.size.split("x")

        ## Returned original_shape is in the format of width, height
        image_resized, origi_shape = load_image_into_numpy_array(
            each_image, int(height), int(width)
        )

        ## The input needs to be a tensor, convert it using `tf.convert_to_tensor`.
        input_tensor = tf.convert_to_tensor(image_resized)

        ## The model expects a batch of images, so add an axis with `tf.newaxis`.
        input_tensor = input_tensor[tf.newaxis, ...]

        ## Feed image into model
        detections_output = trained_model(input_tensor)

        num_detections = int(detections_output.pop("num_detections"))
        detections = {
            key: value[0, :num_detections].numpy()
            for key, value in detections_output.items()
        }
        detections["num_detections"] = num_detections

        ## Filter out predictions below threshold
        indexes = np.where(detections["detection_scores"] > float(args.threshold))

        ## Extract predictions
        bboxes = detections["detection_boxes"][indexes]
        classes = detections["detection_classes"][indexes].astype(np.int64)
        scores = detections["detection_scores"][indexes]

        ## Draw Predictions
        image_origi = Image.fromarray(image_resized).resize(
            (origi_shape[1], origi_shape[0])
        )
        image_origi = np.array(image_origi)

        if len(bboxes) != 0:
            for idx, each_bbox in enumerate(bboxes):
                color = color_map.get(classes[idx] - 1)

                ## Draw bounding box
                cv2.rectangle(
                    image_origi,
                    (
                        int(each_bbox[1] * origi_shape[1]),
                        int(each_bbox[0] * origi_shape[0]),
                    ),
                    (
                        int(each_bbox[3] * origi_shape[1]),
                        int(each_bbox[2] * origi_shape[0]),
                    ),
                    color,
                    2,
                )

                ## Draw label background
                cv2.rectangle(
                    image_origi,
                    (
                        int(each_bbox[1] * origi_shape[1]),
                        int(each_bbox[2] * origi_shape[0]),
                    ),
                    (
                        int(each_bbox[3] * origi_shape[1]),
                        int(each_bbox[2] * origi_shape[0] + 15),
                    ),
                    color,
                    -1,
                )

                ## Insert label class & score
                cv2.putText(
                    image_origi,
                    "Class: {}, Score: {}".format(
                        str(category_index[classes[idx]]["name"]),
                        str(round(scores[idx], 2)),
                    ),
                    (
                        int(each_bbox[1] * origi_shape[1]),
                        int(each_bbox[2] * origi_shape[0] + 10),
                    ),
                    cv2.FONT_HERSHEY_SIMPLEX,
                    0.3,
                    (0, 0, 0),
                    1,
                    cv2.LINE_AA,
                )

            ## Save predicted image
            filename = os.path.basename(each_image)
            image_predict = Image.fromarray(image_origi)
            image_predict.save(os.path.join(args.output, filename))

            print(
                "Saving predicted images to {}...".format(
                    os.path.join(args.output, filename)
                )
            )

if __name__ == "__main__":
    main()


"""Copyright (c) 2012, Daniele Mazzocchio
All rights reserved.

Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:

* Redistributions of source code must retain the above copyright notice, this
  list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
  this list of conditions and the following disclaimer in the documentation
  and/or other materials provided with the distribution.
* Neither the name of the developer nor the names of its contributors may be
  used to endorse or promote products derived from this software without
  specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE."""

"""A light and fast template engine."""

import re

class Template(object):
    """ """

    def __init__(self, template):
        """Initialize class"""
        super(Template, self).__init__()
        self.template = template
        self.options = {"strip": False}

    def expand(self, namespace={}, **kw):
        """Return the expanded template string"""
        namespace.update(kw)
        output = []

        # Builtins
        namespace["echo"]   = lambda s: output.append(s)
        namespace["isdef"]  = lambda v: v in namespace
        namespace["setopt"] = lambda k, v: self.options.update({k: v})

        code = self._process(self._preprocess(self.template))
        eval(compile(code, "<string>", "exec"), namespace)
        return self._postprocess("".join(map(str, output)))

    def _preprocess(self, template):
        """Modify template string before code conversion"""
        # Replace inline ('%') blocks for easier parsing
        o = re.compile("(?m)^[ \t]*%((if|for|while|try).+:)")
        c = re.compile("(?m)^[ \t]*%(((else|elif|except|finally).*:)|(end\w+))")
        template = c.sub(r"<%:\g<1>%>", o.sub(r"<%\g<1>%>", template))

        # Replace (${x}) variables with '<%echo(x)%>'
        v = re.compile("\${([a-zA-Z0-9[\].\"\'_]+)}")
        template = v.sub(r"<%echo(\g<1>)%>\n", template)

        return template

    def _process(self, template):
        """Return the code generated from the template string"""
        code_blk = re.compile(r"<%(.*?)%>\n?", re.DOTALL)
        indent = 0
        code = []
        for n, blk in enumerate(code_blk.split(template)):
            # Replace '<\%' and '%\>' escapes
            blk = re.sub(r"<\\%", "<%", re.sub(r"%\\>", "%>", blk))
            # Unescape '%{}' characters
            blk = re.sub(r"\\(%|{|})", "\g<1>", blk)

            if not (n % 2):
                # Escape double-quote characters
                blk = re.sub(r"\"", "\\\"", blk)
                blk = (" " * (indent*4)) + 'echo("""{}""")'.format(blk)
            else:
                blk = blk.rstrip()
                if blk.lstrip().startswith(":"):
                    if not indent:
                        err = "unexpected block ending"
                        raise SyntaxError("Line {}: {}".format(n, err))
                    indent -= 1
                    if blk.startswith(":end"):
                        continue
                    blk = blk.lstrip()[1:]

                blk = re.sub("(?m)^", " " * (indent * 4), blk)
                if blk.endswith(":"):
                    indent += 1

            code.append(blk)

        if indent:
            err = "Reached EOF before closing block"
            raise EOFError("Line {}: {}".format(n, err))

        return "\n".join(code)

    def _postprocess(self, output):
        """Modify output string after variables and code evaluation"""
        if self.options["strip"]:
            output = re.sub("(?m)(^[ \t]+|[ \t]+$|(?<=[ \t])[ \t]+|^\n)", "",
                            output)

        return output


import boto3
import config

db = boto3.resource('dynamodb', region_name=config.dynamodb_region, endpoint_url=config.dynamodb_url)

table = db.create_table(
    TableName='FLT.Routes',
    KeySchema=[
        {
            'AttributeName': 'route_id',
            'KeyType': 'HASH'
        }
    ],
    AttributeDefinitions=[
        {
            'AttributeName': 'route_id',
            'AttributeType': 'S'
        }
    ],
    ProvisionedThroughput={
        'ReadCapacityUnits': 5,
        'WriteCapacityUnits': 5
    }
)

print("Table status:", table.table_status)


# Defining the word add, so you can add variables later on
def add(a, b):
    print(f"ADDING {a} + {b}")
    return a + b
# Subtract variables later on by defining it
def substract (a, b):
    print(f"SUBSTRACTING {a} - {b}")
    return a - b
# Going to multiply later on by defining it
def multiply(a, b):
    print(f"MULTIPLYING {a} * {b}")
    return a * b
# Will divide the variables by defining it
def divide (a, b):
    print(f"DIVIDING {a} / {b}")
    return a /  b

print("Let's do some math with just functions!")
# Since we defined the functions it will
# do math and it will give you the answer
age = add(30, 5)
height = substract(78, 4)
weight = multiply(90, 2)
iq = divide(100, 2)
# Going to replac whats in the Curly Brackets with the number it got from doing math
print(f"Age: {age}, Height: {height}, Weight: {weight}, IQ: {iq}")

# A puzzle for the etxra credit, type it in anyway.
print("Here is a puzzle.")
# Takes the answers from line 22-25 and uses it to do more math but its all in one line
what = add(age, substract(height, multiply(weight, divide(iq, 2))))

print("That becomes:", what, "Can you do it by hand?")


import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import evaluate as eva
import random
import numpy as np
from sklearn import datasets
from sklearn.cluster import DBSCAN
from sklearn import metrics
from sklearn.datasets.samples_generator import make_blobs
from sklearn.preprocessing import StandardScaler

#二维聚类结果画图
def plot_2D(dict, cluster):
    x = []
    y = []
    for item in dict:
        x.append(dict[item][0])
        y.append(dict[item][1])
    size = 3
    plt.title("DBSCAN Outcome")
    plt.xlabel("death index")
    plt.ylabel("transmission index")
    plt.scatter(x, y, c=cluster, s=size)
    plt.show()

#三维聚类结果画图
def plot_3D(dict, cluster):
    x = []
    y = []
    z = []
    for item in dict:
        x.append(dict[item][0])
        y.append(dict[item][1])
        z.append(dict[item][2])
    ax = plt.subplot(projection='3d')
    plt.title("Cluster Outcome ")
    ax.set_xlabel('death index')
    ax.set_ylabel("transmission index")
    ax.set_zlabel("death speed")
    ax.scatter(x, y, z, c=cluster, s=2)
    plt.draw()
    plt.show()

#计算欧式距离
def distEuclid(x, y):
    return np.sqrt(np.sum((x - y) ** 2))

#判断两点是否在范围内
def isNeighbor(x, y, eps):
    return distEuclid(x, y) <= eps

#获取某一点邻域内的点
def getSeedPos(pos, data, eps):
    seed = []
    for p in range(len(data)):
        if isNeighbor(data[p], data[pos], eps):
            seed.append(p)
    return seed

#获取核心点列表
def getCorePointsPos(data, eps, minpts):
    cpoints = []
    for pos in range(len(data)):
        if len(getSeedPos(pos, data, eps)) >= minpts:
            cpoints.append(pos)
    return cpoints

#DBSCAN聚类步骤
def getCluster(data, eps, minpts):
    corePos = getCorePointsPos(data, eps, minpts)
    unvisited = list(range(len(data)))
    cluster = {}
    num = 0
    for pos in corePos:
        if pos not in unvisited:
            continue
        clusterpoint = []
        clusterpoint.append(pos)
        seedlist = getSeedPos(pos, data, eps)
        unvisited.remove(pos)
        while seedlist:
            p = seedlist.pop(0)
            if p not in unvisited:
                continue
            unvisited.remove(p)
            clusterpoint.append(p)
            if p in corePos:
                seedlist.extend(getSeedPos(p, data, eps))
        cluster[num] = clusterpoint
        num += 1
    cluster[-1] = unvisited
    return cluster

#二维DBSCAN聚类，直接调用库函数，以列表的形式返回聚类结果
def Dcluster_2D(dict):
    points = []
    for item in dict:
        points.append([dict[item][0], dict[item][1]])
    cluster = DBSCAN(eps=0.01, min_samples=9).fit_predict(points)
    return cluster

#三维DBSCAN聚类，用上面实现的函数，以列表的形式返回聚类结果
def Dcluster_3D(dict):
    data = []
    for item in dict:
        data.append(np.array([dict[item][0], dict[item][1], dict[item][2]]))
    clu = getCluster(data, 0.01, 10)
    cluster = [0 for i in range(0, len(dict))]
    for key in clu:
        for point in clu[key]:
            cluster[point] = key
    return cluster

#DBSCAN二维接口函数，主函数调用
def DBSCAN_2D(dict):
    cluster = Dcluster_2D(dict)
    plot_2D(dict, cluster)

#DBSCAN三维接口函数，主函数调用
def DBSCAN_3D(dict):
    cluster = Dcluster_3D(dict)
    plot_3D(dict, cluster)


"""
    Implementation of the Monte Carlo approach for finding approximate solutions to the Rural Postman Problem
    proposed by Cordoba et. al. in this paper:
    http://www.sciencedirect.com/science/article/pii/S0305054898000288

    author: Zach Jones
    date: 4/29/17
"""

from GraphUtils import *
from random import *
import time

verbose = False  # enable/disable verbose mode
g = Graph(filename="test-graphs/dense5")

# configuration variables
max_it = 25
iteration_counter = 0
alpha = 1

# shortest tour found
shortest_tour = None
shortest_tour_path = None
shortest_tour_cost = math.inf

start_time = time.clock()

while iteration_counter < max_it:
    iteration_counter += 1
    # tour[i][j] holds the count of how many times edge (i, j) was traversed
    tour = [[0 for col in range(g.num_vertices)] for row in range(g.num_vertices)]
    tour_path = Path(g)
    edge_required = [[False for col in range(g.num_vertices)] for row in range(g.num_vertices)]
    for edge in g.get_required():
        edge_required[edge[0]][edge[1]] = True
        edge_required[edge[1]][edge[0]] = True

    cost = 0
    num_required = len(g.get_required())
    # probabilities[i][j] holds probability of traversing edge (i, j)
    probabilities = [[0 for col in range(g.num_vertices)] for row in range(g.num_vertices)]
    for v1 in range(0, g.num_vertices):
        for v2 in range(v1, g.num_vertices):
            weight = g.get_weight(v1, v2)
            if weight > 0:
                probabilities[v1][v2] = weight ** -alpha
            else:
                probabilities[v1][v2] = 0

    # randomly place the "vehicle" on a vertex
    current_pos = randint(0, g.num_vertices-1)
    starting_pos = current_pos
    tour_path.extend(starting_pos)
    if verbose: print("Starting at %s" % current_pos)
    while num_required > 0:
        neighbors = g.get_neighbors(current_pos)
        type_1_edges = g.get_type_1_edges(current_pos, edge_required)
        type_2_edges = g.get_type_2_edges(current_pos, edge_required)

        selected_edge = None

        if len(type_1_edges) > 0:
            # randomly pick a type 1 edge to traverse - all T1 edges have equal probability to be selected
            if verbose: print("Choosing type 1 edge")
            selection = randint(0, len(type_1_edges)-1)
            selected_edge = type_1_edges[selection]
        elif len(type_2_edges) > 0:
            # pick a type 2 vertex based on their probabilities
            if verbose: print("Choosing type 2 edge")
            prob_sum = 0
            for edge in type_2_edges:
                prob_sum += probabilities[edge[0]][edge[1]]
            selection = random() * prob_sum
            prob_sum = 0
            for edge in type_2_edges:
                prob_sum += probabilities[edge[0]][edge[1]]
                if selection <= prob_sum:
                    selected_edge = edge
                    break
        else:
            # randomly select a neighboring edge based on their probabilities
            if verbose: print("Choosing another edge")
            prob_sum = 0
            for neighbor in neighbors:
                prob_sum += probabilities[current_pos][neighbor]
            selection = random() * prob_sum
            prob_sum = 0
            for neighbor in neighbors:
                prob_sum += probabilities[current_pos][neighbor]
                if selection <= prob_sum:
                    selected_edge = (current_pos, neighbor)
                    break

        # we have selected an edge to traverse
        if verbose: print("Traversing edge", selected_edge)
        if edge_required[selected_edge[0]][selected_edge[1]]:
            edge_required[selected_edge[0]][selected_edge[1]] = False
            edge_required[selected_edge[1]][selected_edge[0]] = False
            num_required -= 1

        # traverse the edge
        tour[selected_edge[0]][selected_edge[1]] += 1
        tour[selected_edge[1]][selected_edge[0]] += 1
        tour_path.extend(selected_edge[1])
        cost += g.get_weight(selected_edge[0], selected_edge[1])
        current_pos = selected_edge[1]
    # end while

    # we've now traversed all required edges
    # complete the tour by adding a copy of each edge on the shortest path from current_pos to the starting_pos
    shortest_path_to_beginning = djikstra(g, current_pos)[starting_pos]
    for edge in shortest_path_to_beginning.get_edges():
        tour[edge[0]][edge[1]] += 1
        tour[edge[1]][edge[0]] += 1
        tour_path.extend(edge[1])
        cost += g.get_weight(edge[0], edge[1])

    # SR1
    if verbose: print("Cost before SR1: %s" % cost)
    for vertex1 in range(0, len(tour)):
        for vertex2 in range(0, len(tour)):
            while tour[vertex1][vertex2] > 3:
                tour[vertex1][vertex2] -= 2
                tour[vertex2][vertex1] -= 2
                cost -= g.get_weight(vertex1, vertex2)
    if verbose: print("Cost before SR1: %s" % cost)

    # SR2
    # create graph object for this tour used to find connected components - this is not an accurate representation of the tour as a subgraph!
    if verbose: print("Cost before SR2: %s" % cost)
    tour_graph = Graph(adjacency_matrix=tour, required_matrix=[[]], consider_zero_disconnected=True)
    for vertex1 in range(0, len(tour)):
        for vertex2 in range(0, len(tour)):
            if not g.is_required(vertex1, vertex2) and tour[vertex1][vertex2] == 2:
                reachable_before = bfs(tour_graph, vertex1)
                tour_graph.representation[vertex1][vertex2] = None
                reachable_after = bfs(tour_graph, vertex1)
                if len(reachable_after) == len(reachable_before):  # graph is still connected
                    tour[vertex1][vertex2] = 0
                    tour[vertex2][vertex1] = 0
                    cost -= 2 * g.get_weight(vertex1, vertex2)
                else:
                    tour_graph.representation[vertex1][vertex2] = 2

    if verbose: print("Cost after SR2: %s" % cost)

    # SR3
    if verbose: print("Cost before SR3: %s" % cost)
    double_edge = None
    for vertex1 in range(0, len(tour)):
        for vertex2 in range(0, len(tour)):
            if tour[vertex1][vertex2] == 2:
                double_edge = (vertex1, vertex2)
                break
    if double_edge is not None:
        double_path = Path(tour_graph, double_edge[0])
        double_path.extend(double_edge[1])
        expansion_found = True
        while expansion_found:
            end_vertex = double_path.vertices[-1]
            neighbors = tour_graph.get_neighbors(end_vertex)
            expansion_found = False
            for neighbor in neighbors:
                if neighbor not in double_path.vertices and tour[end_vertex][neighbor] == 2:
                    expansion_found = True
                    double_path.extend(neighbor)
                    break
        # remove one copy of each edge in double_path and add an edge along the shortest path
        start_vertex = double_path.vertices[0]
        end_vertex = double_path.vertices[-1]
        for edge in double_path.get_edges():
            tour[edge[0]][edge[1]] -= 1
            tour[edge[1]][edge[0]] -= 1
            cost -= g.get_weight(edge[0], edge[1])
        for edge in djikstra(g, start_vertex)[end_vertex].get_edges():
            tour[edge[0]][edge[1]] += 1
            tour[edge[1]][edge[0]] += 1
            cost += g.get_weight(edge[0], edge[1])

    if verbose: print("Cost after SR3: %s" % cost)

    if verbose: print("Found tour with cost %s" % cost)

    if cost < shortest_tour_cost:
        shortest_tour = tour
        shortest_tour_path = tour_path
        shortest_tour_cost = cost

# end outer while

time_elapsed = time.clock() - start_time

print("Done searching!\nBest tour found has cost %s" % shortest_tour_cost)
print("Path before simplification routines:\n %s" % shortest_tour_path)

print("Time elapsed: %s ms" % (time_elapsed*1000))


from poetry.console import console
from poetry.console.commands.command import Command

class InstallCommand(Command):
    """
    install project dependencies

    install
        {--sync : Synchronize the environment with the locked packages and the specified groups.}
        {--dry-run : Output the operations but do not execute anything (implicitly enables --verbose).}
        {--extras=* : Extra sets of dependencies to install.}
        {--update : update the given packages to the last compatible version }
        {--lock-only : Do not perform operations (only update the lockfile). }
        {--editable : Add vcs/path dependencies as editable.}
        {--optional : add packages as an optional dependencies. }
        {--python= : Python version for which the dependency must be installed. }
        {--platform= : Platforms for which the dependency must be installed. }
        {--source= : Name of the source to use to install the package.}
        {--allow-prereleases : Accept prereleases.}
        {packages?* : The packages to add. }
    """

    help = """
    install project dependencies
    If you do not specify a version constraint, rp will choose a suitable one based on the available package versions.

    You can specify a package in the following forms:
      - A single name (requests)
      - A name and a constraint (requests@^2.23.0)
      - A git url (git+https://github.com/python-poetry/poetry.git)
      - A git url with a revision (git+https://github.com/python-poetry/poetry.git#develop)
      - A git SSH url (git+ssh://github.com/python-poetry/poetry.git)
      - A git SSH url with a revision (git+ssh://github.com/python-poetry/poetry.git#develop)
      - A file path (../my-package/my-package.whl)
      - A directory (../my-package/)
      - A url (https://example.com/packages/my-package-0.1.0.tar.gz)
    """

    def handle(self) -> int:
        from poetry.app.relaxed_poetry import rp

        project = rp.active_project

        for subp in project.projects_graph():
            if subp.env:
                subp.install(
                    self.argument("packages"),
                    synchronize=self.option("sync"),
                    dry_run=self.option("dry-run"),
                    extras_strings=self.option("extras"),
                    update=self.option("update"),
                    lock_only=self.option("lock-only"),
                    editable=self.option("editable"),
                    optional=self.option("optional"),
                    python=self.option("python"),
                    platform=self.option("platform"),
                    source=self.option("source"),
                    allow_prereleases=self.option("allow-prereleases")
                )
            else:
                console.println(
                    f"<info>Skipping {subp.pyproject.name}, it does not requires python interpreter and therefore cannot have dependencies.</>\n"
                    "To change that, add a python dependency to <c1>pyproject.toml</c1>")

        return 0


from lxml import etree
from pytest import set_trace
from DocketParse import grammar_modules
from glob import iglob
import os

def revise_section(docket_tree, section_name, section_module):
  """
  Method is used for plucking an unparsed section out of a parsed docket.
  This is useful when dockets have already been completely parsed with certain
  sections, but you'd like to parse another section but avoid parsing
  everything from the pdfs again.

  Input: the etree of the incompletely parsed docket,
         the name of the section to be revised,
         the name of the module to parse the section.
  Output: An etree now with the revised section parsed.
  """
  __import__("DocketParse.grammar_modules." + section_module)
  module = getattr(grammar_modules, section_module)

  root = docket_tree.getroot()
  docket_num = root.xpath("/docket/header/docket_number/text()")
  section_to_parse = root.xpath("/docket/section[@name='{}']/text()".format(section_name))[0]
  print("Section_to_parse:")
  print(section_to_parse)
  try:
    #set_trace()
    parsed_section = module.parse(section_to_parse)
  except:
    raise Exception("Could not parse section {} for {}.".format(section_name, docket_num))

  root.xpath("/docket/section[@name='{}']".format(section_name))[0].text = ""
  root.xpath("/docket/section[@name='{}']".format(section_name))[0].append(etree.fromstring(parsed_section))
  return docket_tree

def add_suffix(file, suffix):
  split_file = os.path.splitext(file)
  return "{}{}{}".format(split_file[0], suffix, split_file[1])

def revise_directory(dir_path, section_name, section_module, suffix=""):
  """
  Processes an entire directory of dockets that have an unparsed section to
  parse.
  Input: 1) path to a directory with dockets that have a section needing to be
            parsed.
         2) Name of section to be parsed and replaced.
         3) Name of grammar module
         4) Suffix. Optional.  Suffix is added to the revised dockets, so if
            you want to keep old and new versions, add a suffix to prevent the
            new version from overwriting the old one.
  """
  iterator = iglob("{}*.xml".format(dir_path))
  counter = 0
  successes = 0
  for file in iterator:
    print("Revising {}".format(file))
    try:
      tree = revise_section(etree.parse(file), section_name, section_module)
      successes += 1
      new_file = add_suffix(file, suffix)
      with open(new_file, "w") as f:
        f.write(etree.tostring(tree, pretty_print=True).decode("utf-8"))
      f.close()
    except:
      print("{} failed revision.".format(file))
    counter += 1
  return (counter, successes)

#End


# -*- coding: utf-8 -*-

from flask import Blueprint, render_template, redirect, url_for, request, send_file
from pythosong import ql, calc
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.orm.exc import NoResultFound

import os.path
import io
import datetime

blueprint = Blueprint('pythosong', __name__)

@blueprint.route('/')
@blueprint.route('/index')
def index():
    return render_template('pythosong/index.html')

@blueprint.route('/works')
def works():
    return render_template('pythosong/works.html')

@blueprint.route('/work/<int:id>')
def work(id):
    return render_template('pythosong/work.html',
                           performer=ql.get_work_performer(id),
                           artists=ql.get_work_artists(id),
                           names=ql.get_work_names(id),
                           charts=ql.get_work_charts(id),
                           data=ql.get_work_data(id))

@blueprint.route('/performers')
def performers():
    return render_template('pythosong/performers.html')

@blueprint.route('/performer/<int:id>')
def performer(id):
    return render_template('pythosong/performer.html',
                           names=ql.get_performer_names(id),
                           artists=ql.get_performer_artists(id),
                           works=ql.get_performer_works(id),
                           performer=ql.get_performer(id=id))

@blueprint.route('/artists')
def artists():
    return render_template('pythosong/artists.html')

@blueprint.route('/artists_youtube')
def artists_youtube():
    return render_template('pythosong/artists_youtube.html')

@blueprint.route('/artist/<int:id>')
def artist(id):
    return render_template('pythosong/artist.html',
                           artist=ql.get_artist_data(id),
                           performers=ql.get_artist_performers(id),
                           works=ql.get_artist_works(id))

@blueprint.route('/calculate')
def calculate():
    calc.run_all()
    return redirect(url_for('pythosong.index'))

@blueprint.route('/add_youtube/<int:work_id>', methods=['GET', 'POST'])
def add_youtube(work_id):

    if request.method == 'POST':
        youtube_url = request.form['youtube_url']
        ql.set_youtube(work_id=work_id, url=youtube_url)

    return render_template('pythosong/add_youtube.html', data=ql.get_work_data2(work_id))

@blueprint.route('/merge_performer/', methods=['POST'])
def merge_performer():
    old_id = request.form['old_id']
    new_id = request.form.get('new_id', None)
    new_name = request.form.get('new_name', None)

    assert old_id
    assert new_id or new_name

    if new_name:
        try:
            new_id = ql.performer_name_to_id(new_name)
        except SQLAlchemyError as e:
            e.message += 'No performer found with name (%s)' % new_name
            raise

    assert new_id != old_id
    ql.join_performers(old_id=old_id, new_id=new_id)

    return redirect(url_for('pythosong.performer', id=new_id))

@blueprint.route('/add_performer_artist/', methods=['POST'])
def add_performer_artist():
    performer_id = request.form['performer_id']
    artist_id = request.form.get('artist_id', None)
    artist_name = request.form.get('artist_name', None)

    assert performer_id
    assert artist_id or artist_name

    if artist_name:
        try:
            artist_id = ql.artist_name_to_id(artist_name)
        except NoResultFound as e:
            e.args = ('No artist found with name (%s)' % artist_name,)
            raise

    assert not ql.exists_performer_artist(performer_id, artist_id)
    ql.add_performer_artist(performer_id, artist_id)

    return redirect(request.referrer)

@blueprint.route('/add_performer_name/', methods=['POST'])
def add_performer_name():
    id = request.form['id']
    name = request.form['name']

    assert id
    assert name

    ql.add_performer_name(id, name)

    return redirect(request.referrer)

@blueprint.route('/set_artist_name/', methods=['POST'])
def set_artist_name():
    id = request.form['id']
    name = request.form['name']

    assert id
    assert name

    ql.set_artist_name(id, name)

    return redirect(request.referrer)

@blueprint.route('/set_artist_country/', methods=['POST'])
def set_artist_country():
    id = request.form['id']
    country = request.form['country']

    assert id
    assert country

    ql.set_artist_country(id, country)

    return redirect(request.referrer)

@blueprint.route('/add_work_name/', methods=['POST'])
def add_work_name():
    id = request.form['id']
    name = request.form['name']

    assert id
    assert name

    ql.add_work_name(id, name)

    return redirect(request.referrer)

@blueprint.route('/set_performer_main_name/<int:performer_id>/<int:name_id>')
def set_performer_main_name(performer_id, name_id):
    assert performer_id
    assert name_id

    ql.set_performer_main_name(performer_id=performer_id, name_id=name_id)

    return redirect(request.referrer)

@blueprint.route('/set_work_main_name/<int:work_id>/<int:name_id>')
def set_work_main_name(work_id, name_id):
    assert work_id
    assert name_id

    ql.set_work_main_name(work_id=work_id, name_id=name_id)

    return redirect(request.referrer)

@blueprint.route('/merge_works/<int:old_id>/<int:new_id>')
def merge_works(old_id, new_id):
    assert old_id
    assert new_id

    ql.merge_works(old_id=old_id, new_id=new_id)

    return ('', 204)

@blueprint.route('/remove_performer_name/<int:performer_id>/<int:name_id>')
def remove_performer_name(performer_id, name_id):
    assert performer_id
    assert name_id

    ql.remove_performer_name(performer_id=performer_id, name_id=name_id)

    return redirect(request.referrer)

@blueprint.route('/remove_performer_artist/<int:performer_id>/<int:artist_id>')
def remove_performer_artist(performer_id, artist_id):
    assert performer_id
    assert artist_id

    ql.remove_artist_from_performer(performer_id=performer_id, artist_id=artist_id)

    return redirect(url_for('pythosong.performer', id=performer_id))

@blueprint.route('/delete_artist/<int:id>')
def delete_artist(id):
    assert id
    ql.delete_artist(id)
    return redirect(url_for('pythosong.artists'))

@blueprint.route('/delete_performer/<int:id>')
def delete_performer(id):
    assert id
    ql.delete_performer(id)
    return redirect(url_for('pythosong.performers'))

@blueprint.route('/cleanup')
def cleanup():
    ql.cleanup_performers_artist()
    return redirect(request.referrer)

@blueprint.route('/set_main_artist/<int:performer_id>/<int:artist_id>/<int:main_artist>')
def set_main_artist(performer_id, artist_id, main_artist):
    assert performer_id
    assert artist_id
    assert main_artist in (0, 1)

    ql.set_main_artist(performer_id=performer_id, artist_id=artist_id, main_artist=(main_artist==1))

    return redirect(request.referrer)

@blueprint.route('/set_work_performer', methods=['POST'])
def set_work_performer():
    work_id = request.form['work_id']
    performer_id = request.form.get('performer_id', None)
    performer_name = request.form.get('performer_name', None)

    assert work_id
    assert performer_id or performer_name

    if performer_name:
        try:
            performer_id = ql.performer_name_to_id(performer_name)
        except SQLAlchemyError as e:
            e.message += 'No performer found with name (%s)' % performer_name
            raise

    ql.set_work_performer(work_id=work_id, performer_id=performer_id)

    return redirect(request.referrer)

@blueprint.route('/check_artist_codes')
def check_artist_codes():

    dict_codes = dict()
    dict_chars = dict()
    for row in ql.get_artists():
        code = codify(row.name, dict_chars)
        if code not in dict_codes:
            dict_codes[code] = []
        dict_codes[code].append((row.id, row.name, row.score if row.score else 0))

    list_chars = [(k, v) for k, v in dict_chars.items()]
    if list_chars:
        for l in sorted(list_chars, key=lambda l: l[1], reverse=True):
            print(l[0].encode('ascii', 'xmlcharrefreplace'))
            print(l[1])
        raise ValueError('Unknown characters found - please view terminal and add characters to codify.')

    list_doubles = []
    for code, artists in dict_codes.items():
        if len(artists) > 1:
            list_doubles.append(dict(code=code, artists=artists, score=sum(artist[2] for artist in artists)))
    list_doubles.sort(key=lambda el: el['score'], reverse=True)

    return render_template('pythosong/check_artist_codes.html', doubles=list_doubles)

@blueprint.route('/check_performer_codes')
def check_performer_codes():

    dict_codes = dict()
    dict_chars = dict()
    for row in ql.get_performers():
        code = codify(row.name, dict_chars)
        if code not in dict_codes:
            dict_codes[code] = []
        dict_codes[code].append((row.id, row.name, row.score if row.score else 0))

    list_chars = [(k, v) for k, v in dict_chars.items()]
    if list_chars:
        for l in sorted(list_chars, key=lambda l: l[1], reverse=True):
            print(l[0].encode('ascii', 'xmlcharrefreplace'))
            print(l[1])
        raise ValueError('Unknown characters found - please view terminal and add characters to codify.')

    list_doubles = []
    for code, performers in dict_codes.items():
        if len(performers) > 1:
            list_doubles.append(dict(code=code, performers=performers, score=sum(performer[2] for performer in performers)))
    list_doubles.sort(key=lambda el: el['score'], reverse=True)

    return render_template('pythosong/check_performer_codes.html', doubles=list_doubles)

@blueprint.route('/check_work_codes')
def check_work_codes():

    works = dict()
    new_chars = dict()

    for work in ql.get_all_work_names():
        key = (work.performer_id, work.type, codify(work.name, new_chars, is_performer=False))
        value = work.work_id

        if key not in works:
            works[key] = set()
        works[key].add(value)

    list_chars = [(k, v) for k, v in new_chars.items()]
    if list_chars:
        for l in sorted(list_chars, key=lambda l: l[1], reverse=True):
            print(l[0].encode('ascii', 'xmlcharrefreplace'))
            print(l[1])
        raise ValueError('Unknown characters found - please view terminal and add characters to codify.')

    list_doubles = [list(k) + [v] for (k,v) in works.items() if len(v) > 1]

    data = []
    for double in list_doubles:
        performer_id = double[0]
        performer_name = ql.get_performer_names(performer_id=performer_id)['name_main']['name']
        type = double[1]
        code = double[2]
        works = []
        for work_id in double[3]:
            work_db = ql.get_work_data(work_id)
            works.append(dict(id=work_id, score=(work_db.score if work_db.score else 0) + (10000 if work_db.youtube else 0), name=ql.get_work_names(work_id=work_id)['name_main']['name']))
        works.sort(key=lambda el: el['score'], reverse=True)
        data.append(dict(performer_id=performer_id, performer_name=performer_name, type=type, code=code, works=works))

    return render_template('pythosong/check_work_codes.html', data=data)

@blueprint.route('/check_double_works')
def check_double_works():
    return render_template('pythosong/check_double_works.html', doubles=ql.get_double_works())

@blueprint.route('/check_double_artist_in_work')
def check_double_artist_in_work():
    return render_template('pythosong/check_double_artist_in_work.html', doubles=ql.get_double_artist_in_work())

@blueprint.route('/works_with_youtube')
def works_with_youtube():
    return render_template('pythosong/works_with_youtube.html', works=ql.get_works_with_youtube())

@blueprint.route('/download_works_with_youtube')
def download_works_with_youtube():
    works = ql.get_works_with_youtube()

    rows = []
    for work in works:
        try:
            dl_url = 'http://www.youtubeinmp3.com/fetch/?video=%s' % work.youtube
            print(dl_url)

            target = calc.get_download_filename(work_name=work.name, performer_name=work.performer_name)
            print(target)

            calc.download_file(url=dl_url, target=target)
            row = ['Uitvoerder?', '%s (%s)' % (work.performer_name, work.name), 'Audiovraag', work.date, os.path.basename(target)]
            ql.set_youtube(work_id=work.id, url='ok')
        except BaseException as e:
            row = [unicode(type(e))] + list(e.args)

        rows.append(row)

    file_stream = io.BytesIO()
    calc.save_list_of_lists_to_excel_stream(rows, file_stream)
    file_stream.seek(0)
    return send_file(file_stream,
                     attachment_filename='dlmp3_%s.xlsx' % datetime.datetime.now().strftime('%y%m%d_%H%M%S'),
                     as_attachment=True)

@blueprint.route('/allow_double_artist_in_work/<int:artist_id>/<int:work1_id>/<int:work2_id>')
def allow_double_artist_in_work(artist_id, work1_id, work2_id):
    assert artist_id
    assert work1_id
    assert work2_id

    ql.allow_double_artist_in_work(artist_id=artist_id, work1_id=work1_id, work2_id=work2_id)
    return redirect(request.referrer)

@blueprint.route('/new_artist/<string:name>')
def new_artist(name):
    assert name
    ql.new_artist(name)
    return ('', 204)

@blueprint.route('/new_performer/<string:name>')
def new_performer(name):
    assert name
    ql.new_performer(name)
    return ('', 204)

@blueprint.route('/upload')
def upload():
    return render_template('pythosong/upload.html')

@blueprint.route('/upload_works', methods=['POST'])
def upload_works():
    ret = calc.process_upload_works(request.files['excel'])
    if ret:
        file_stream = io.BytesIO()
        calc.save_performers_to_stream(ret, file_stream)
        file_stream.seek(0)
        return send_file(file_stream, attachment_filename='performers_%s.xlsx' % datetime.datetime.now().strftime('%y%m%d_%H%M%S'),
                         as_attachment=True)
    return upload()

@blueprint.route('/upload_performers', methods=['POST'])
def upload_performers():
    calc.process_upload_performers(request.files['excel'])
    return upload()

import re

def codify(name, chars, is_performer=True):
    name = name.lower()

    if is_performer:
        if name.startswith('de '):
            name = name[3:]
        elif name.startswith('the '):
            name = name[4:]

        for removed in (' en ', ' and ', ' & ', ' et ', ' feat. ', ' feat ', ' ft. ', ' feat ', ' with ', ' met ', ' pres. '):
            name = name.replace(removed, '')

    replacements = [(u'!._-? \'\"()[],+:/*&@|·#%¡‘•°´;»•⦜•^=><{}~\\`’…¿–¬¤†÷¦£«‰★❤承–', ''),
                    (u'àäáåâãª', 'a'),
                    (u'ç©¢ć', 'c'),
                    (u'éèêë', 'e'),
                    (u'íïìî', 'i'),
                    (u'öóòôºọ', 'o'),
                    (u'øœ', 'oe'),
                    (u'$š§', 's'),
                    (u'üùúû', 'u'),
                    (u'ÿý¥','y')
                    ]

    for (olds, new) in replacements:
        for char in olds:
            name = name.replace(char, new)

    name = (name.replace(u'æ', 'ae')
            .replace(u'ß', 'ss')
            .replace(chr(160), "")
            .replace(chr(156), "oe")
            .replace(u'²', '2')
            .replace(u'³', '3')
            .replace(u'þ', 'th')
            .replace(u'ð', 'd')
            .replace(u'ñ', 'n')
            .replace(chr(351), 's')
            .replace(u'µ', 'mu')
            .replace(u'½', '12')
            .replace(u'ž', 'z')
            .replace(chr(149), "")
            .replace(chr(150), "")
            .replace(chr(299), "")
            .replace(chr(8220), "")
            .replace(chr(8221), "")
            .replace(chr(8730), "")
            .replace(chr(8706), "")
            .replace(chr(174), "")
	    .replace(chr(49900), "")
            )
    for char_id in (32080, 36681, 65533, 54616, 49900, 47803, 51652, 51204, 50896, 51648, 45209, 54620):
        name = name.replace(chr(char_id), "")

    for c in name:
        if not re.match('[a-z0-9]', c):
            if c not in chars:
                chars[c] = 0
            chars[c] += 1

    return name

from pythosong import api
# blueprint.open_resource('file.txt') opent bestand in root path van blueprint


from rest_framework import generics, mixins, status, viewsets
from rest_framework.exceptions import NotFound
from rest_framework.generics import UpdateAPIView
from rest_framework.permissions import (
    AllowAny, IsAuthenticated, IsAuthenticatedOrReadOnly, IsAdminUser
)
from rest_framework.response import Response
from rest_framework.views import APIView
from django.db.models import Avg, Count

from .models import Article, ArticleRating
from .renderers import ArticleJSONRenderer, RatingJSONRenderer
from .serializers import ArticleSerializer, RatingSerializer
from rest_framework.pagination import LimitOffsetPagination
from django.conf import settings

from django_filters.rest_framework import DjangoFilterBackend
import django_filters
from rest_framework.filters import SearchFilter
from django.views.generic import ListView
import django_filters
from django_filters import rest_framework as filters
from django.contrib.postgres.fields import ArrayField
from .models import Article, Comment, Report, ArticleRating
from .renderers import ArticleJSONRenderer, CommentJSONRenderer, ReportJSONRenderer
from .serializers import ArticleSerializer, CommentSerializer, ReportSerializer

class ArticleViewSet(mixins.CreateModelMixin,
                     mixins.ListModelMixin,
                     mixins.RetrieveModelMixin,
                     mixins.DestroyModelMixin,
                     viewsets.GenericViewSet):
    ''' By subclassing create, list, retrieve and destroy
    we can define create, list, retrieve and destroy
    endpoints in one class '''

    lookup_field = 'slug'
    queryset = Article.objects.all()
    permission_classes = (IsAuthenticatedOrReadOnly,)
    renderer_classes = (ArticleJSONRenderer,)
    serializer_class = ArticleSerializer
    pagination_class = LimitOffsetPagination

    def get_queryset(self):
        queryset = self.queryset

        return queryset

    def create(self, request):

        ''' Creates a new article in the database
        Method takes user data, validates and commits in the db
        '''

        serializer_context = {
            'author': request.user,
            'request': request
        }
        serializer_data = request.data.get('article', {})

        serializer = self.serializer_class(
            data=serializer_data, context=serializer_context
        )
        serializer.is_valid(raise_exception=True)
        serializer.save(author=request.user)

        return Response(serializer.data,
 \
                        status=status.HTTP_201_CREATED)

    def list(self, request):

        ''' Retrives all articles from the database
        with the latest to be created first
        (chronologically)
        '''

        serializer_context = {'request': request}
        page = self.paginate_queryset(self.queryset)

        serializer = self.serializer_class(
            page,
            context=serializer_context,
            many=True
        )

        return self.get_paginated_response(serializer.data)

    def retrieve(self, request, slug):

        ''' Method returns a single article
        Takes a slug as unique identifier, searches the db
        and returns an article with matching slug.
        Returns NotFound if an article does not exist '''

        serializer_context = {'request': request}

        try:
            article = Article.objects.get(slug=slug)
        except Article.DoesNotExist:

            raise NotFound('An article with this slug does not exist.')

        serializer = self.serializer_class(
            article,
            context=serializer_context
        )

        return Response(
            {'article': serializer.data}, status=status.HTTP_200_OK)

    def update(self, request, slug):

        ''' Method updates partially a single article
        Takes a slug as unique identifier, searches the db
        and updates an article with matching slug.
        Returns NotFound if an article does not exist '''

        serializer_context = {'request': request}

        try:
            serializer_instance = self.queryset.get(slug=slug)
        except Article.DoesNotExist:

            raise NotFound('An article with this slug does not exist.')

        serializer_data = request.data.get('article', {})

        serializer = self.serializer_class(
            serializer_instance,
            context=serializer_context,
            data=serializer_data,
            partial=True
        )
        article = Article.objects.get(slug=slug);

        if request.user != article.author:
            return Response(
                {'message': 'You can only update your article'},

                status=status.HTTP_401_UNAUTHORIZED)

        serializer.is_valid(raise_exception=True)
        serializer.save()

        return Response(serializer.data, status=status.HTTP_200_OK)

    def destroy(self, request, slug):

        ''' Method deletes a single article
        Takes a slug as unique identifier, searches the db
        and deletes an article with matching slug.
        Returns NotFound if an article does not exist '''

        serializer_context = {'request': request}

        try:
            serializer_instance = self.queryset.get(slug=slug)

        except Article.DoesNotExist:
            raise NotFound('An article with this slug does not exist.')

        article = Article.objects.get(slug=slug);

        if request.user != article.author:
            return Response({'message': 'You can only delete your article'},
                            status=status.HTTP_401_UNAUTHORIZED)

        if article.delete():
            return Response(
                {'message': 'You have successfully deleted the article'},
                status=status.HTTP_200_OK)

class LikeAPIView(UpdateAPIView):
    queryset = Article.objects.all()
    permission_classes = (IsAuthenticated,)
    renderer_classes = (ArticleJSONRenderer,)
    serializer_class = ArticleSerializer

    lookup_field = 'slug'

    def update(self, request, slug):
        """Update the user's liking status on a particular article."""
        # if not auth Sign in to make your opinion count.
        user = request.user

        try:
            article = Article.objects.get(slug=slug)
        except Exception:
            return Response({"message": "The article does not exist."}, status=status.HTTP_404_NOT_FOUND)
            # raise NotFound('The article does not exist.')

        if user in article.like.all():
            # If like exists, remove like
            article.like.remove(user.id)
            return Response({"message": "You no longer like this article"}, status=status.HTTP_200_OK)
        if user in article.dislike.all():
            # If user had disliked the article, removes the dislike and adds the like
            article.dislike.remove(user.id)
            article.like.add(user.id)
            return Response({"message": "Removed from dislike and Added to Liked articles"}, status=status.HTTP_200_OK)
        else:
            # If like does not exist, add like
            article.like.add(user.id)
            return Response({"message": "Added to Liked articles"}, status=status.HTTP_200_OK)

class DisLikeAPIView(UpdateAPIView):
    queryset = Article.objects.all()
    permission_classes = (IsAuthenticated,)
    renderer_classes = (ArticleJSONRenderer,)
    serializer_class = ArticleSerializer

    lookup_field = 'slug'

    def update(self, request, slug):
        """Update the user's liking status on a particular article."""
        # if not auth Sign in to make your opinion count.
        user = request.user

        try:
            article = Article.objects.get(slug=slug)
        except Exception:
            return Response({"message": "The article does not exist."}, status=status.HTTP_404_NOT_FOUND)

        if user in article.dislike.all():
            # If dislike exists, remove dislike
            article.dislike.remove(user.id)
            return Response({"message": "You no longer dislike this article"}, status=status.HTTP_200_OK)
        if user in article.like.all():
            article.like.remove(user.id)
            article.dislike.add(user.id)
            return Response({"message": "Removed from Liked Articles and Added to Disliked articles"},
                            status=status.HTTP_200_OK)
        else:
            # If dislike does not exist, add dislike
            article.dislike.add(user.id)
            return Response({"message": "You Dislike this Article"}, status=status.HTTP_200_OK)

class ArticleFilter(filters.FilterSet):
    """
    Class creates a custom filter class for articles,
    for getting dynamic queries from the url
    """
    title = filters.CharFilter(field_name='title', lookup_expr='icontains')
    description = filters.CharFilter(
        field_name='description', lookup_expr='icontains')
    body = filters.CharFilter(field_name='body', lookup_expr='icontains')
    author__username = filters.CharFilter(
        field_name='author__username', lookup_expr='icontains')

    class Meta:
        """
        This class describes the fields to be used in the search.
        The ArrayField has also been over-ridden
        """
        model = Article
        fields = [
            'title', 'description', 'body', 'author__username', 'tagList'
        ]
        filter_overrides = {
            ArrayField: {
                'filter_class': django_filters.CharFilter,
                'extra': lambda f: {
                    'lookup_expr': 'icontains', },
            },
        }

class ArticleSearchList(generics.ListAPIView):
    """
    Implements class to enable searching and filtering
    """

    permission_classes = (IsAuthenticatedOrReadOnly,)
    search_list = ['title', 'body',
                   'description', 'author__username', 'tagList']
    filter_list = ['title', 'author__username', 'tagList', ]
    queryset = Article.objects.all()
    serializer_class = ArticleSerializer
    filter_backends = (DjangoFilterBackend, SearchFilter,)
    filter_fields = filter_list
    search_fields = search_list
    filterset_class = ArticleFilter

class ArticlesFavoriteAPIView(APIView):
    """
    This View allows users to Favorite and Unfavorite articles, an exception is
    thrown if the article doesn’t
    exist.
    """
    permission_classes = (IsAuthenticated,)
    renderer_classes = (ArticleJSONRenderer,)
    serializer_class = ArticleSerializer

    def delete(self, request, article_slug=None):
        """
        Unfavorites an existing article with the profile unfavorite method
        """
        profile = self.request.user.profile
        serializer_context = {'request': request}

        try:
            article = Article.objects.get(slug=article_slug)
        except Article.DoesNotExist:
            raise NotFound('An article with this slug was not found.')

        profile.unfavorite(article)

        serializer = self.serializer_class(article, context=serializer_context)

        return Response(serializer.data, status=status.HTTP_200_OK)

    def post(self, request, article_slug=None):
        """
        unfavorites an existing article with the profile favorite method
        """
        profile = self.request.user.profile
        serializer_context = {'request': request}

        try:
            article = Article.objects.get(slug=article_slug)
        except Article.DoesNotExist:
            raise NotFound('An article with this slug was not found.')

        profile.favorite(article)

        serializer = self.serializer_class(article, context=serializer_context)

        return Response(serializer.data, status=status.HTTP_200_OK)

class CommentsListCreateAPIView(generics.ListCreateAPIView):
    '''
    View to create and list comments
    '''

    lookup_field = 'article__slug'
    lookup_url_kwarg = 'slug'

    queryset = Comment.objects.select_related()
    serializer_class = CommentSerializer
    renderer_classes = (CommentJSONRenderer,)
    permission_classes = (IsAuthenticatedOrReadOnly,)

    def filter_queryset(self, queryset):
        """Handle getting comments on an article."""
        filters = {self.lookup_field: self.kwargs[self.lookup_url_kwarg]}
        return queryset.filter(**filters)

    def get(self, request, **kwargs):
        '''
        Get all comments to an article
        '''
        slug = self.kwargs['slug']
        try:
            article = Article.objects.get(slug=slug)
            comments = Comment.objects.all().filter(article=article.id)
            serializer = self.serializer_class(comments, many=True)

            return Response(serializer.data,
                            status=status.HTTP_200_OK)
        except Article.DoesNotExist:

            raise NotFound('An article with this slug does not exist.')

    def create(self, request, **kwargs):
        '''
        Add comment to an article
        '''
        slug = self.kwargs['slug']
        try:
            article = Article.objects.get(slug=slug)
        except Article.DoesNotExist:
            raise NotFound("An article with that slug does not exist")
        serializer_context = {"author": request.user, "slug": slug}
        serializer_data = request.data.get('comment', {})
        serializer = self.serializer_class(
            data=serializer_data,
            context=serializer_context)
        serializer.is_valid(raise_exception=True)
        serializer.save()

        return Response(serializer.data, status=status.HTTP_201_CREATED)

class CommentRetrieveUpdateDestroy(generics.RetrieveUpdateDestroyAPIView):
    '''
    View to retreive, update and delete a single comment
    '''
    lookup_url_kwarg = 'pk'
    serializer_class = CommentSerializer
    permission_classes = (IsAuthenticatedOrReadOnly,)
    queryset = Comment.objects.all()

    def retrieve(self, request, **kwargs):
        '''
        Get single comment
        '''
        slug = self.kwargs['slug']
        pk = self.kwargs['pk']
        try:
            article = Article.objects.get(slug=slug)
        except Article.DoesNotExist:

            raise NotFound('An article with this slug does not exist.')
        try:
            comment = Comment.objects.get(pk=pk)
        except:
            raise NotFound('The comment with that id does not exist')

        serializer = self.serializer_class(comment)

        return Response({"comment": serializer.data}, status=status.HTTP_200_OK)

    def destroy(self, request, **kwargs):
        '''
        Delete comment
        '''
        pk = self.kwargs['pk']
        try:
            comment = Comment.objects.get(pk=pk)
        except Comment.DoesNotExist:
            raise NotFound('A comment with this ID does not exist.')

        if comment.author.id != request.user.id:
            return Response({
                "error": "you can not delete a comment that does not belong to you"},
                status=status.HTTP_401_UNAUTHORIZED)
        comment.delete()

        return Response({
            "message": "Your comment has been successfully deleted"},
            status=status.HTTP_200_OK)

class LikeComment(APIView):
    '''
    Like a comment
    '''
    serializer_class = CommentSerializer
    permission_classes = (IsAuthenticatedOrReadOnly,)
    renderer_classes = (CommentJSONRenderer,)

    def put(self, request, **kwargs):
        '''
        update like field in comment model
        '''
        slug = self.kwargs['slug']
        pk = self.kwargs['pk']
        try:
            article = Article.objects.get(slug=slug)
        except Article.DoesNotExist:
            raise NotFound('An article with this slug does not exist.')
        try:
            comment = Comment.objects.get(pk=pk)
        except Comment.DoesNotExist:
            raise NotFound('A comment with this ID does not exist.')

        comment.dislikes.remove(request.user)

        if request.user in comment.likes.all():
            comment.likes.remove(request.user)
            return Response({"message": "You unliked this article."},
                            status=status.HTTP_200_OK)

        comment.likes.add(request.user)
        return Response({"message": "You liked this comment"},
                        status=status.HTTP_200_OK)

class DislikeComment(APIView):
    '''
    dislike a comment
    '''
    serializer_class = CommentSerializer
    permission_classes = (IsAuthenticatedOrReadOnly,)
    renderer_classes = (CommentJSONRenderer,)

    def put(self, request, **kwargs):
        '''
        update dislike field in comment model
        '''
        slug = self.kwargs['slug']
        pk = self.kwargs['pk']
        try:
            article = Article.objects.get(slug=slug)
        except Article.DoesNotExist:

            raise NotFound('An article with this slug does not exist.')
        try:
            comment = Comment.objects.get(pk=pk)
        except Comment.DoesNotExist:
            raise NotFound('A comment with this ID does not exist.')

        comment.likes.remove(request.user)

        if request.user in comment.dislikes.all():
            comment.dislikes.remove(request.user)
            return Response({"message": "You undisliked this article."},
                            status=status.HTTP_200_OK)

        comment.dislikes.add(request.user)
        return Response({"message": "You disliked this comment"},
                        status=status.HTTP_200_OK)

class RateArticlesAPIView(APIView):
    permission_classes = (IsAuthenticatedOrReadOnly,)
    renderer_classes = (RatingJSONRenderer,)
    serializer_class = RatingSerializer

    def post(self, request, slug):
        """
        Method that posts users article ratings
        """
        ratingData = request.data.get("rate", {})

        rating = ratingData.get('rating', None)
        note = ratingData.get('note', None)

        if not isinstance(rating, int):
            return Response(
                {'message': 'Rating should be an integer'},
                status=status.HTTP_401_UNAUTHORIZED)
        if rating <1 or rating > 5:
            return Response(
                {'error': 'Rating should be between 1 and 5'},
                status=status.HTTP_401_UNAUTHORIZED)
        # Try to get an article using slug, raise an exception if not found
        try:
            article = Article.objects.get(slug=slug)
        except Article.DoesNotExist:
            raise NotFound("An article with this slug does not exist")
        # User should not rate own article, raise unauthorised status
        if request.user ==  article.author:
            return Response(
                {'message': 'You cannot rate your article'},

                status=status.HTTP_401_UNAUTHORIZED)
        # No previous ratings, so just take the rating and save to the db
        # after aggregating
        # Check the existing number or rating
        # SELECT count(ID) FROM RATINGS WHERE ARTICLE = article and Rater = user
        user = request.user
        try:
            # Get a user rating if there is previous record
            currentRating = ArticleRating.objects.get(rater=user, article=article);
        except:
            currentRating = None

         # Save the rating
        if currentRating is None:
            article_rating = ArticleRating(article=article,
            rating=rating, rater=user, note=note)
            article_rating.save()
        ## update the rating
        else:
            #Upadte previous rating with current rating
            currentRating.rating=rating
            currentRating.save()

        # Get the ratings avarage
        average = ArticleRating.objects.filter(
            article=article).aggregate(Avg('rating')).get('rating__avg', 0)
        return Response(
            {"average_rating": average, "Note": note},
            status=status.HTTP_201_CREATED)

                    # Get the ratings avarage
        average = ArticleRating.objects.filter(
            article=article).aggregate(Avg('rating')).get('rating__avg', 0)
        return Response(
            {"average_rating": average, "Note": note},
            status=status.HTTP_201_CREATED)
    # End pints returns an initial rating of the user if exists
    def get(self, request, slug):
        """
        Method that posts users article ratings
        """

        # Try to get an article using slug, raise an exception if not found
        try:
            article = Article.objects.get(slug=slug)
        except Article.DoesNotExist:
            raise NotFound("An article with this slug does not exist")
        # after aggregating
        # Check the existing number or rating
        # SELECT count(ID) FROM RATINGS WHERE ARTICLE = article and Rater = user
        user = request.user
        try:
            # Get a user rating if there is previous record
            currentRating = ArticleRating.objects.get(
                rater=user, article=article)
        except:
            currentRating = None
            return Response("You have not rated before")

         # Return the rating
        if currentRating:
            return Response(
                {"rating": currentRating.rating},
                status=status.HTTP_200_OK)

class BookmarkAPIView(UpdateAPIView):
    permission_classes = (IsAuthenticated,)
    renderer_classes = (ArticleJSONRenderer,)
    serializer_class = ArticleSerializer

    lookup_field = 'slug'

    def update(self, request, slug):
        """Update the bookmark status on a particular article."""
        user = request.user
        profile = user.profile

        # Checks if the article exists
        try:
            article = Article.objects.get(slug=slug)
        except Article.DoesNotExist:
            raise NotFound('An article with this slug does not exist.')

        bookmarks = user.profile.bookmarks.all()
        # Bookmarks article, if article is not bookmarked
        if article not in bookmarks:
            user.profile.bookmarks.add(article)
            response = 'Added to Bookmarks'
        # Remove bookmark, if one exists
        else:
            user.profile.bookmarks.remove(article)
            response = 'Removed from Bookmarks'
        return Response(response, status=status.HTTP_200_OK)

class ReportCreateAPIView(generics.CreateAPIView):
    """Facilitate create reports"""

    lookup_field = 'article__slug'
    lookup_url_kwarg = 'slug'

    queryset = Report.objects.select_related()
    serializer_class = ReportSerializer
    renderer_classes = (ReportJSONRenderer,)
    permission_classes = (IsAuthenticatedOrReadOnly,)

    def filter_queryset(self, queryset):
        """Handle getting reports on an article."""
        filters = {self.lookup_field: self.kwargs[self.lookup_url_kwarg]}
        return queryset.filter(**filters)

    def create(self, request, **kwargs):
        """Create reports to an article"""
        slug = self.kwargs['slug']
        try:
            article = Article.objects.get(slug=slug)
        except Article.DoesNotExist:
            raise NotFound("An article with that slug does not exist")
        serializer_context = {"author": request.user, "slug": slug}
        serializer_data = request.data.get('report', {})
        serializer = self.serializer_class(data=serializer_data, context=serializer_context)
        serializer.is_valid(raise_exception=True)
        serializer.save()

        return Response(serializer.data, status=status.HTTP_201_CREATED)

class ReportListAPIView(generics.ListAPIView):
    """Facilitate list reports"""

    lookup_field = 'article__slug'
    lookup_url_kwarg = 'slug'

    queryset = Report.objects.select_related()
    serializer_class = ReportSerializer
    renderer_classes = (ReportJSONRenderer,)
    permission_classes = (IsAdminUser,)

    def filter_queryset(self, queryset):
        """Handle getting reports on an article."""
        filters = {self.lookup_field: self.kwargs[self.lookup_url_kwarg]}
        return queryset.filter(**filters)

    def get(self, request, **kwargs):
        """Get all reports to an article"""
        slug = self.kwargs['slug']
        try:
            article = Article.objects.get(slug=slug)
            reports = Report.objects.all().filter(article=article.id)
            serializer = self.serializer_class(reports, many=True)

            return Response(serializer.data, status=status.HTTP_200_OK)
        except Article.DoesNotExist:

            raise NotFound('An article with this slug does not exist.')


# Sample Stacked denoising autoencoder (2 layers)
# Note use of chicken_selects module and how to process data imports
# Written by Woochan H. (Last modified: 26/03/18)

import torch
import torch.nn as nn
from torch.autograd import Variable
import numpy as np
import matplotlib.pyplot as plt
from chicken_selects import *

# Object Data('model type', 'motion', noiselevel, cuda = False)
data = Data('Stacked Autoencoder', 'flexion_extension', 1)

if torch.cuda.is_available == True:
    data.cuda_on()

# Specify directory if you have changed folder name / dir
data.set_ecg_filepath()
data.set_emg_filepath()
data.set_acc_filepath()

# Call data into numpy array format. Check soure code for additional input specifications
clean_ecg = data.pull_all_ecg(t0 = 0, tf = 30000)
noisy_ecg = data.pull_all_emg()
acc_dat = data.pull_all_acc()

# Acc data modified to fit that of noisy emg
acc_dat = np.array(list(acc_dat[:, 0:3000].transpose())*140).transpose()
# Clean generated from gaussian dist, N(0, 0.05)
clean_acc = np.random.randn(np.shape(acc_dat)[0], np.shape(acc_dat)[1]) *0.05

# Add ACC data onto clean/noisy ecg data
input_dat = np.vstack((noisy_ecg, acc_dat))
label_dat = np.vstack((clean_ecg, clean_acc))

# Reformat to shape that can be imported to neural net
input_dat = data.reformat(input_dat, feature_len = 200, data_form = 1)
label_dat = data.reformat(label_dat, feature_len = 200, data_form = 1)
print("Input Data shape: {}".format(np.shape(input_dat)))
print("Label Data shape: {}".format(np.shape(label_dat)))
print("Step 0: Data Import Done")

#input_dat = data.undo_reformat(input_dat)
#label_dat = data.undo_reformat(label_dat)

_ = input("Continue(y/n)?: ")

# Change numpy array to tensor variable
Tensor_Input = Variable(torch.from_numpy(input_dat).float())
Tensor_Label = Variable(torch.from_numpy(label_dat).float())

# Hyper Parameters
EPOCH = int(input("Epochs?: "))
LR = float(input("Learning rate?: "))
BATCH_SIZE = int(input("Batch size?: "))

# Autoencoder Model Structure
class StackedAutoEncoder(nn.Module):
    def __init__(self):
        super(StackedAutoEncoder, self).__init__()

        self.encoder1 = nn.Sequential(
            nn.Linear(800, 270),
            nn.Tanh(),
            nn.Linear(270, 135),
            nn.Tanh(),
            nn.Linear(135, 70),
        )
        self.decoder1 = nn.Sequential(
            nn.Linear(70, 135),
            nn.Tanh(),
            nn.Linear(135, 270)
        )
        self.encoder2 = nn.Sequential(
            nn.Linear(270, 135),
            nn.Tanh(),
            nn.Linear(135, 70),
        )
        self.decoder2 = nn.Sequential(
            nn.Linear(70, 135),
            nn.Tanh(),
            nn.Linear(135, 270),
            nn.Tanh(),
            nn.Linear(270, 800)
        )

    def forward(self, x):
        en1 = self.encoder1(x)
        de1 = self.decoder1(en1)
        en2 = self.encoder2(de1)
        de2 = self.decoder2(en2)
        return en1, de1, en2, de2

print("Step 1: Model Setup Done")

# Setting of Loss function and optimizer
SAE = StackedAutoEncoder()
optimizer = torch.optim.Adam(SAE.parameters(), lr=LR)
loss_func = nn.MSELoss()
train_loss = []

if torch.cuda.is_available() == True:
    SAE = SAE.cuda()

def save_params(save_name):
    dir = '{}/Trained_Params/{}/{}'.format(data.filepath, data.model, save_name)
    if not os.path.exists(dir):
        os.makedirs(dir)
    torch.save(SAE.encoder1, dir + '/encoder1_{}.pt'.format(EPOCH))
    torch.save(SAE.decoder1, dir + '/decoder1_{}.pt'.format(EPOCH))
    torch.save(SAE.encoder2, dir + '/encoder2_{}.pt'.format(EPOCH))
    torch.save(SAE.decoder2, dir + '/decoder2_{}.pt'.format(EPOCH))
    np.save(dir + '/trainloss_{}.npy'.format(EPOCH),train_loss)
    print("Step 3: Model Saved")

# Train the model
try:
    print("Step 2: Model Training Start")

    for epoch in range(EPOCH):

        en1, de1, en2, de2 = SAE(Tensor_Input)

        loss = loss_func(de2, Tensor_Label)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        print('Epoch: ', epoch + 1, '| train loss: %.4f' % loss.data[0])
        train_loss.append(loss.data[0])

    print("Step 2: Model Training Finished")

    # Plot Loss
    plt.figure(figsize = (10,4));
    plt.plot(train_loss, color='k', linewidth=0.4, linestyle='-', label = 'MSE loss');
    plt.legend(loc = 2);
    plt.title("Training Loss({} | {} | LR:{})".format(data.model, data.motion, LR));
    plt.xlabel("Epochs")
    plt.ylabel("Loss")

    plt.show()

    # Save trained Parameters
    if str(input("Save Parameters?(y/n): ")) == 'y':
        save_name = str(input("Save parameters as?: ")) + '_Interrupted'
        save_params(save_name)
        print("End of Session")
    else:
        print("Session Terminated. Parameters not saved")

except KeyboardInterrupt:

    if str(input("Save Parameters?(y/n): ")) == 'y':
        save_name = str(input("Save parameters as?: ")) + '_Interrupted'
        save_params(save_name)
    else:
        print("Session Terminated. Parameters not saved")


#!/usr/bin/python3
'''
Exports data to CSV
'''
import csv, os, requests, sys

def main(argv):

    emp = requests.get('https://jsonplaceholder.typicode.com/users/{}'
                       .format(sys.argv[1]))
    name = emp.json().get('username')
    tasks = requests.get('https://jsonplaceholder.typicode.com/todos')
    tasks = tasks.json()
    filename = sys.argv[1] + ".csv"
    with open(filename, 'w') as f:
        writer = csv.writer(f, quoting=csv.QUOTE_ALL, quotechar='"')
        for task in tasks:
            if task['userId'] == int(sys.argv[1]):
                task['name'] = name
                writer.writerow([task['userId'], task['name'],
                                 task['completed'], task['title']])

if __name__ == "__main__":
    import sys
    main(sys.argv)


import bs4
from urllib.request import urlopen as uReq
from bs4 import BeautifulSoup as soup
for page in range(1,39):
 my_url = "https://stackoverflow.com/jobs"
 uClient = uReq(my_url)
 page_html = uClient.read()
 uClient.close()
 page_soup=soup(page_html,"html.parser")
 page_var=page_soup.findall("div",{"class":"-title"})
 for i in page_var:
    job_name =i.findall("a",{"class":"s-link s-link__visited"})
    skills_required =job_name[i].findall("a",{"class":"post-tag job-link no-tag-menu"})
    print(job_name.string, end="------------")
    print(skills_required.string, end=" ")
    print("\n")


shoppinglist=['milk','pasta','eggs','spam','rice']
itemtofind='albestross'
foundat=None   ##initially=0(NULL)

for index in range(len(shoppinglist)):
    if shoppinglist[index]==itemtofind:
        foundat=index
        break
if foundat is not None:
    print('item found at {}'.format(foundat))
else:
    print("{} not found".format(itemtofind))

print('*'*800)

       ##########          OR
shoppinglist=['milk','pasta','eggs','spam','rice']
itemtofind='spam'
foundat=None   ##initially=0(NULL)
if itemtofind in shoppinglist:
    foundat=shoppinglist.index(itemtofind)

if foundat is not None:
    print('item found at {}'.format(foundat))
else:
    print("{} not found".format(itemtofind))


import numpy as np
import matplotlib
# matplotlib.use("Agg")
import matplotlib.pyplot as plt

# fig = plt.figure(figsize=(6, 3), dpi=400)
from scipy import signal

maxi = 1900
dt = 1

# vx = np.ndarray((maxi, 300, 550))
# ux = np.ndarray((maxi, 300, 550))
# vz = np.ndarray((maxi, 300, 550))
# uz = np.ndarray((maxi, 300, 550))
# for i in range(0, maxi):
#     vx[i, :, :] = np.transpose(np.loadtxt("snapshots/snapshot%i_vx.txt" % i))
#     vz[i, :, :] = np.transpose(np.loadtxt("snapshots/snapshot%i_vz.txt" % i))
#
#     ux[i, :, :] = ux[i - 1, :, :] + dt * vx[i, :, :]
#     uz[i, :, :] = uz[i - 1, :, :] + dt * vz[i, :, :]

vx = np.transpose(np.loadtxt("snapshots/snapshot%i_vx.txt" % maxi))
vz = np.transpose(np.loadtxt("snapshots/snapshot%i_vz.txt" % maxi))

maxAmp = 1e-10

dz = np.array([[0, -1, 0],
               [0, 0, 0],
               [0, 1, 0]])
dx = np.array([[0, 0, 0],
               [-1, 0, 1],
               [0, 0, 0]])

field1 = vx  # ux[-1, :, :]
field2 = vz  # uz[-1, :, :]

uxdz = signal.convolve2d(field1, dz, boundary='symm', mode='same')
uzdx = signal.convolve2d(field2, dx, boundary='symm', mode='same')

uxdx = signal.convolve2d(field1, dz, boundary='symm', mode='same')
uzdz = signal.convolve2d(field2, dx, boundary='symm', mode='same')

# plt.subplot(3, 1, 1)
# im = plt.imshow(field1[:-50, 50:-50], cmap=plt.get_cmap('seismic'), vmin=-maxAmp, vmax=maxAmp)
# # im = plt.imshow(uxdz[:-50, 50:-50] - uzdx[:-50, 50:-50], cmap=plt.get_cmap('seismic'), vmin=-maxAmp, vmax=maxAmp)
# plt.subplot(3, 1, 2)
# plt.imshow(field2[:-50, 50:-50], cmap=plt.get_cmap('seismic'), vmin=-maxAmp, vmax=maxAmp)
# # plt.imshow(uxdx[:-50, 50:-50] + uzdz[:-50, 50:-50], cmap=plt.get_cmap('seismic'), vmin=-maxAmp, vmax=maxAmp)
# plt.subplot(3, 1, 3)
field3 = (np.sqrt(np.square(field1[:-50, 50:-50]) + np.square(field2[:-50, 50:-50])))
plt.imshow(np.log10(field3), cmap=plt.get_cmap('Reds'), vmin=-12, vmax=-5)
plt.show()


import requests
import json
import time
import emoji
import ast
import re

class Crawling:
    bearer_token = ""
    search_url = ""
    query_params = {}

    def __init__(self):
        # To set your environment variables in your terminal run the following line:
        # export 'BEARER_TOKEN'='<your_bearer_token>'
        self.bearer_token = "AAAAAAAAAAAAAAAAAAAAAMjyOwEAAAAAYm8zDZ9NMtvCd4K%2FTckEVsozHCA%3Daj3fsQ8Pgmka7tfdzLkeTFcY8jtvZW0hhDps7UFVnsiU1aJhnv"
        self.search_url = "https://api.twitter.com/2/tweets/search/all"
        self.search_url_from_id = "https://api.twitter.com/2/users/"

        # Optional params: start_time,end_time,since_id,until_id,max_results,next_token,
        # expansions,tweet.fields,media.fields,poll.fields,place.fields,user.fields
        self.query_params = {'query': '', 'tweet.fields': 'created_at,lang,author_id',
                              'start_time': '2016-01-01T00:00:00Z', 'end_time': '2021-01-01T00:00:00Z', 'max_results': '500'}
        self.query_params_id = {'tweet.fields': 'created_at,lang,author_id,geo', 'start_time': '2016-01-01T00:00:00Z', 'end_time': '2021-01-01T00:00:00Z',
                                'max_results': '100'}

        # expansions=author_id&tweet.fields=created_at,author_id,conversation_id,public_metrics,context_annotations&user.fields=username&max_results=5

    def create_headers(self):
        headers = {"Authorization": "Bearer {}".format(self.bearer_token)}
        return headers

    def connect_to_endpoint(self, headers):
        response = requests.request("GET", self.search_url, headers=headers, params=self.query_params)
        # print(response.status_code)
        if response.status_code != 200:
            raise Exception(response.status_code, response.text)
        return response.json()

    def main_act_indi(self, brand_list, drug_name, start_time, end_time):
        self.query_params['start_time'] = start_time
        self.query_params['end_time'] = end_time
        self.query_params['tweet.fields'] = "lang,created_at,author_id"
        self.query_params['expansions'] = "geo.place_id"
        self.query_params['place.fields'] = "country"
        # main activity
        fw = open(drug_name + " " + self.query_params['start_time'][0:10] + "~" + self.query_params['end_time'][
                                                                                0:10] + ".json", "w", encoding="UTF8") # illegal multibyte sequence 오류 -> UTF8 설정으로 고침
        for brand_name in brand_list:
            self.query_params['query'] = brand_name
            print(self.query_params)
            self.crawling_part(fw)

    ''' start crawling
     parameter : 크롤링 결과가 담길 json 파일 객체 '''
    def crawling_part(self, fw):
        headers = self.create_headers()

        # crawling 후 json_response에 twit 데이터 저장
        json_response = self.connect_to_endpoint(headers)
        ## twit text 문장 안에 '/" 있을 시 json으로 파싱 에러 -> 제거해줌
        for i in range(0, len(json_response['data'])):
            json_response['data'][i]['text'] = json_response['data'][i]['text'].replace("'", "").replace('"', "")

        # 이모티콘/느낌표,물음표 제거 (이모티콘 제거 안할 시 -> encoding 에러 발생)
        json_response = str(json_response)
        json_response = emoji.get_emoji_regexp().sub(r'', json_response)
        json_response = re.sub(r'[^ 0-9ㄱ-ㅣ가-힣A-Za-z.,"\':;_\-@(){}\[\]]', '', json_response)
        json_response = ast.literal_eval(json_response)

        # json 파싱
        data = json.dumps(json_response, indent=4, sort_keys=True)
        data = data.encode('utf-8')
        # 한글 출력 위한 decode
        data = data.decode('unicode_escape', 'replace')

        fw.write(data)
        try:
            self.query_params['next_token'] = json_response['meta']['next_token']
        except Exception as e:
            print(self.query_params['query'] + " done")
            self.query_params['next_token'] = "DUMMY"
            del self.query_params['next_token']
            # print(e)
            time.sleep(3)
            return
        time.sleep(3)
        self.crawling_part(fw)


from PyQt5 import QtCore, QtGui, QtWidgets
from Eage import Ui_Dialog
#from PyQt5.QtWidgets import QFileDialog
#from imageedit import imageedit_self
#from bottombutton import button_self
from PyQt5.QtCore import *
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *
import cv2, os

cwd = os.getcwd()

class mywindow(QtWidgets.QMainWindow, Ui_Dialog):
    def  __init__ (self):
        super(mywindow, self).__init__()
        self.setupUi(self)
        self.OpenButton.clicked.connect(self.read_file)#打开
        self.ProcessButton.clicked.connect(self.process_file)#处理
        self.SaveButton.clicked.connect(self.save_file)#保存
        self.CameraButton.clicked.connect(self.camera_open)#打开相机

    def read_file(self):
        filename, filetype =QtWidgets.QFileDialog.getOpenFileName(self, "打开文件", cwd+"\\input\\", "All Files(*)")# ;;Text Files(*.png)
        self.img = cv2.imread(filename)
        cv2.imwrite(cwd+"\\input\\1.jpg",self.img)
        imgshow = QtGui.QPixmap(filename).scaled(self.inputimage.width(), self.inputimage.height())
        self.inputimage.setPixmap(imgshow)

    def process_file(self):
        os.chdir(cwd)
        # os.system(r"cd C:\Users\11323\Desktop\design\age-estimation-pytorch-master&&python demo.py --img_dir input --output_dir output")
        os.system(r"python demo.py --img_dir input --output_dir output")

        filename = cwd+"\\output\\1.jpg"
        imgshow = QtGui.QPixmap(filename).scaled(self.inputimage.width(), self.inputimage.height())
        self.outputimage.setPixmap(imgshow)

    def save_file(self):
        filename=QtWidgets.QFileDialog.getSaveFileName(self,'save file',cwd+"\\output\\")
        # print(filename)
        self.img2 = cv2.imread(cwd+"\\output\\1.jpg")
        cv2.imwrite(filename[0]+".jpg",self.img2)

    def camera_open(self):
        os.chdir(cwd)
        os.system(r"python demo.py")

if __name__=="__main__":
    import sys
    app=QtWidgets.QApplication(sys.argv)
    ui = mywindow()
    ui.show()
    app.exec_()