import cv2
import numpy as np
from scipy.spatial import distance as dist
from sklearn.cluster import DBSCAN
from sklearn import metrics
from sklearn.datasets.samples_generator import make_blobs
from sklearn.preprocessing import StandardScaler
from sklearn.datasets.samples_generator import make_blobs
from sklearn.cluster import AffinityPropagation
from sklearn.metrics import *
try:
    from PIL import Image
except ImportError:
    import Image
import pytesseract

def label(img, mask, contours):
    cv2.imshow('mask', mask)
    cv2.waitKey(0)

    mask = cv2.erode(mask, None, iterations=2)
    meanHSV = cv2.mean(img, mask=mask)[:3]
    #print(meanHSV)

img = cv2.imread('image path')
#cv2.imshow('result01', img)
cv2.waitKey(0)

img_hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
#cv2.imshow('result02', img_hsv)
cv2.waitKey(0)

img_gray = cv2.cvtColor(img_hsv, cv2.COLOR_BGR2GRAY)
#cv2.imshow('result03', img_gray)
cv2.waitKey(0)

edges = cv2.Canny(img_gray,50,150,apertureSize=3)

#그외 전처리

#laplacian = cv2.Laplacian(blur, cv2.CV_8U, ksize=7)
#laplacian2 = cv2.Laplacian(laplacian, cv2.CV_64F)
#cv2.imshow('result04',laplacian)
#cv2.waitKey(0)

lines = cv2.HoughLines(edges,2,np.pi/30,100)

#허프 transform으로 detect된 line을 보이고 싶을 때

if lines is not None :
    for i in range(len(lines)):

        for rho, theta in lines[i]:

            a = np.cos(theta)
            b = np.sin(theta) # r = xcos+ysin
            x0 = a*rho #rho : 매개 변수의 해상도
            y0 = b*rho #시작점

            x1 = int(x0 + 1000*(-b))#np.cos(theta)*rho - 1000*(np.sin(theta))
            y1 = int(y0+1000*(a))
            x2 = int(x0 - 1000*(-b))
            y2 = int(y0 -1000*(a))
            #100개 이상 vote 된 점

            deg = np.rad2deg(theta)
            arr = np.empty(len(lines))
            arr[i] = deg
            #print(rho)
            #print(arr)

            cv2.line(img,(x1,y1),(x2,y2),(0,0,255),2)
            # 즉 점(x1,y1)과 점(x2,y2) 은 하나의 같은 점 선 위에 있다.

            # 클러스터링
            X = lines

            X = np.reshape(X, (-1, 2))
            model = AffinityPropagation(preference=-100).fit(X)

            cluster_centers_indices = model.cluster_centers_indices_
            labels = model.labels_
            n_clusters_ = len(cluster_centers_indices)

            # print('Estimated number of clusters: %d' % n_clusters_)

    if len(lines) >= 1:
        for i in range(len(lines)):  #

            for rho, theta in lines[i]:  #
                deg = np.rad2deg(theta)
                arr = np.empty(len(lines))
                arr[i] = deg
                # print(rho)
                #print(arr)

    if n_clusters_ == 3:

        print("triangle")

    elif n_clusters_ == 4:  # size == 4 :
                print("rectangle")

h, s, v = cv2.split(img_hsv)
#cv2.imshow('result04', v)
cv2.waitKey(0)

blur = cv2.GaussianBlur(img_gray, (5, 5), 0)
ret, img_binary = cv2.threshold(blur, 0, 255, cv2.THRESH_BINARY_INV | cv2.THRESH_OTSU)

#cv2.imshow('result05', img_binary)
cv2.waitKey(0)

width, height = img_binary.shape[:2]

if img_binary[0, 0] == 255:
    for i in range(width):
        for j in range(height):
            if img_binary[i][j] == 0:
                img_binary[i][j] = 255
            else:
                img_binary[i][j] = 0

#cv2.imshow('result05-1', img_binary)
cv2.waitKey(0)

contours, img2 = cv2.findContours(img_binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)

for cnt in contours:
        size = len(cnt)
        #print(size)
        #n_clusters_ = len(cluster_centers_indices)

        epsilon = 0.01 * cv2.arcLength(cnt, True)
        approx = cv2.approxPolyDP(cnt, epsilon, True)

        size = len(approx)

        #print(size)

        cv2.line(img, tuple(approx[0][0]), tuple(approx[size - 1][0]), (0, 255, 0), 3)
        for k in range(size - 1):
            cv2.line(img, tuple(approx[k][0]), tuple(approx[k + 1][0]), (0, 255, 0), 3)

        if cv2.isContourConvex(approx):

            if size == 5:
                print("pentagon")
            elif size == 6:
                print("hexagon")
            elif size == 8:
                print("octagon")
            else:

                x, y, w, h = cv2.boundingRect(cnt)
                aspect_ratio = float(w) / h
                print("Aspect Ratio : {}".format(aspect_ratio))

                if aspect_ratio >= 1.2 or aspect_ratio <= 0.9:

                    if lines is not None :

                        print("Blockyform")

                        if arr[0] < 85 or arr[0] > 95:

                            # 회전
                            matrix = cv2.getRotationMatrix2D((img.shape[0], img.shape[1]), -90 + arr[0], 1)
                            dst = cv2.warpAffine(img, matrix, (img.shape[0] * 2, img.shape[1] * 2))

                    else: print("ellipse")
                else:

                 if lines is None:
                    print("circle")

#cv2.imshow('result06', img)
cv2.waitKey(0)

## (4) Create mask and do bitwise-op
for cnt in contours:
    mask = np.zeros(img_hsv.shape[:2], np.uint8)
    cv2.drawContours(mask, [cnt], -1, 255, -1)
    dst = cv2.bitwise_and(img_hsv, img_hsv, mask=mask)  # bitwiseand

#cv2.imshow('result07', dst)
cv2.waitKey(0)

h, s, v = cv2.split(img_hsv)
mean = cv2.mean(img_hsv,mask=mask)[:3]
#print(mean)

#h,s,v 평균.

h = mean[0]
s = mean[1]
v = mean[2]

#분류
if 0<h<15 or 165<h<180: #붉은 계열

        if s<73 : print ("pink")
        elif 80>s>73 : print("purple")
        elif 180>s>140 :print("red")
        elif 255 > v > 200:print("white")
        elif 10 > v > 0:
            print("black")

        else : print("brown")

elif 15<h<75: #초록 계열
    if s<60 :print("orange")
    elif 125>s>70: print ("green")
    elif 130>s>163:print("yellowgreen")

    else: print("yellow")

elif 75<h<165: #푸른 계열
    if s<45 : print("cyan")
    elif 80<s<15 : print("blue")

    else : print("indigo")

elif s<10 :
    if v > 200 : print("white")
    if v < 10 : print ("black")

else : print("null")

colors = [[166, 172, 100], [192, 77, 218], [146, 84, 191], [6, 73, 209], [7, 148, 171], [48, 162, 175], [6, 183, 80],
          [25, 59, 224],
          [95, 44, 145], [146, 84, 191], [68, 117, 170], [25, 168, 230], [24, 8, 19], [80, 4, 208],[11 ,167, 151]]
colorNames = ["indigo", "purple", "blue", "pink", "red", "yellowgreen", "violet", "orange", "cyan", "blue", "green",
              "yellow", "black", "white","brown"]

cv2.waitKey()

#tesseract OCR
print(pytesseract.image_to_string('image path'))


#!/usr/bin/python3

import os
import glob
import time
from typing import List

import cv2
import numpy as np

from keras.optimizers import Adam

from .unet import unet
from .utils.metric_utils import dice_coef, dice_coef_loss
from .utils.img_processing_utils import binarize_img, mkdir_s

weights_path = os.path.realpath(__file__)
weights_path = weights_path.replace(
    "robin/binarize.py",
    "weights/bin_weights.hdf5")

def main(
    input: str = os.path.join(".", "input"),
    output: str = os.path.join(".", "output"),
    weights_path: str = weights_path,
    batchsize: int = 2,
) -> List[np.ndarray]:
    """Binarize images from input directory and
    write them to output directory.

    Parameters
    ----------
    input: str, optional
        input path for images
    output: str, optional
        output path to save images
    weights_path: str, optional
        path to weights file
    batchsize: int, optional
        batchsize to use in model prediction

    Retuns
    ------
    List[numpy.ndarray]
        list of binary images in np.ndarray format

    Notes
    -----
    All input image names should be in png format "sample_1.png".
    All output image names will end with "_bin" like "sample_1_bin.png".

    Example
    -------
    robin.binarize.main('input_path', 'output_path', 2)

    """
    try:
        assert (batchsize > 0) and isinstance(batchsize, int)
    except Exception:
        print("batchsize should be > 0 and int but given {}".format(batchsize))
    try:
        assert (os.path.isdir(input))
    except Exception:
        print("Input path is not valid")

    start_time = time.time()

    fnames_in = list(glob.iglob(os.path.join(
        input,
        "**",
        "*.png*"), recursive=True))
    model = None
    if len(fnames_in) != 0:
        mkdir_s(output)
        model = unet()
        model.compile(
            optimizer=Adam(lr=1e-4),
            loss=dice_coef_loss,
            metrics=[dice_coef])
        model.load_weights(weights_path)

    bin_img_list = []

    for fname in fnames_in:
        print("binarizing -> {0}".format(fname))
        img = cv2.imread(fname, cv2.IMREAD_GRAYSCALE).astype(np.float32)
        img = binarize_img(img, model, batchsize)
        cv2.imwrite(
            os.path.join(
                output,
                os.path.split(fname)[-1].replace(".png", "_bin.png")),
            img
        )
        bin_img_list.append(img)
    print("finished in {0:.2f} seconds".format(time.time() - start_time))
    return bin_img_list

if __name__ == "__main__":
    main()


import os

import torch

class CheckPoint(object):

    def __init__(self, root, save_fn, load_fn, record_filename="checkpoint.txt", num_record=-1, clean=False):
        self.root = root
        self.save_fn = save_fn
        self.load_fn = load_fn
        self.filename = record_filename
        self.num_record = num_record

        if not os.path.exists(root):
            os.makedirs(root)

        # clean the record file
        record_filename = os.path.join(self.root, self.filename)
        if clean and os.path.exists(record_filename):
            with open(record_filename, "w") as f:
                f.write('')

    def read_record_file(self):
        record_filename = os.path.join(self.root, self.filename)
        if not os.path.exists(record_filename):
            return []
        with open(record_filename, "r") as f:
            records = f.read().split()
            records = [i for i in records if i.strip()]
            return records

    def write_record_file(self, records):
        record_filename = os.path.join(self.root, self.filename)
        with open(record_filename, "w") as f:
            for record in records:
                f.write("{}\n".format(record))

    def save(self, filename, *args):
        if not os.path.isabs(filename):
            filename = os.path.join(self.root, filename)
        self.save_fn(filename, *args)
        records = self.read_record_file()
        records.append(filename)

        if 0 < self.num_record < len(records):
            while self.num_record < len(records):
                record = records.pop(0)
                try:
                    os.remove(record)
                except Exception:
                    pass

        self.write_record_file(records)

    def load(self, filename):
        self.load_fn(filename)

    def try_load_last(self):
        record_filename = os.path.join(self.root, self.filename)
        if not os.path.exists(record_filename):
            return
        with open(record_filename, "r") as f:
            records = f.read().split()
            records = [i for i in records if i.strip()]

        if len(records) > 0:
            self.load_fn(records[-1])

def search_checkpoint_records(save_dir):
    checkpoint_files = []
    if os.path.exists(save_dir):
        checkpoint_record = os.path.join(save_dir, "checkpoint.txt")
        if os.path.exists(checkpoint_record):
            for line in open(checkpoint_record):
                if line.strip() != "":
                    # print(line.strip())
                    checkpoint_files.append(line.strip())
    while len(checkpoint_files) > 5:
        try:
            print("remove {}".format(checkpoint_files[0]))
            os.remove(os.path.join(save_dir, checkpoint_files.pop(0)))
        except Exception:
            pass

    return checkpoint_files

def save_checkpoint_records(checkpoint_files, save_dir):
    with open(os.path.join(save_dir, "checkpoint.txt"), "w") as f:
        for checkpoint_file in checkpoint_files:
            f.write("{}\n".format(checkpoint_file))

def save_checkpoint(model, optimizer, scheduler, epoch, path):
    save_dir = os.path.dirname(path)
    checkpoint_files = search_checkpoint_records(save_dir)
    if not os.path.exists(save_dir):
        os.makedirs(save_dir)
    if len(checkpoint_files) == 5:
        try:
            os.remove(os.path.join(save_dir, checkpoint_files.pop(0)))
        except Exception:
            pass
    checkpoint_files.append(os.path.basename(path))
    torch.save({
        "epoch": epoch,
        "model_state_dict": model.state_dict(),
        "optimizer_state_dict": optimizer.state_dict(),
        "scheduler_state_dict": scheduler.state_dict(),
    }, path)

    save_checkpoint_records(checkpoint_files, save_dir)

def load_checkpoint(save_path):
    checkpoint_path = save_path
    if os.path.isdir(save_path) and os.path.exists(save_path):
        checkpoint_records = search_checkpoint_records(save_path)
        if len(checkpoint_records) == 0:
            raise FileNotFoundError(os.path.join(save_path, "checkpoint.txt"))
        checkpoint_path = os.path.join(save_path, checkpoint_records[-1])
    print("restore checkpoint from {}".format(checkpoint_path))
    if os.path.exists(checkpoint_path):
        return torch.load(checkpoint_path, map_location="cpu")
    else:
        raise FileNotFoundError(checkpoint_path)

def log_loss(losses, save_dir, epoch, mode="train"):
    assert isinstance(losses, list)
    if not os.path.exists(save_dir):
        os.makedirs(save_dir)
    with open(os.path.join(save_dir, "{}.txt".format(mode)), "a") as f:
        f.write("{} ".format(epoch))
        for loss in losses:
            f.write("{} ".format(loss))
        f.write("\n")


""" Face Color Machine Learning code """
from collections import Counter
import pprint
import base64
from io import BytesIO
from facenet_pytorch import MTCNN, InceptionResnetV1, extract_face
from PIL import Image, ImageDraw
import numpy as np
import cv2
from sklearn.cluster import KMeans
import imutils
from matplotlib import pyplot as plt
#from matplotlib.figure import Figure
#path = 'backend/ml/image/anne-marie'
plt.switch_backend('Agg')

resnet = InceptionResnetV1(pretrained='vggface2').eval()

def img_to_face(image_path):
    if not isinstance(image_path, str):
        final_path = image_path
    else:
        final_path = 'media/' + str(image_path)
    img = Image.open(final_path).convert('RGB')
    mtcnn = MTCNN(image_size=100)
    # Get cropped and prewhitened image tensor
    img_cropped = mtcnn(img, save_path='media/output/face.png')
    # Calculate embedding (unsqueeze to add batch dimension)
    img_embedding = resnet(img_cropped.unsqueeze(0))
    # Or, if using for VGGFace2 classification
    resnet.classify = True
    img_probs = resnet(img_cropped.unsqueeze(0))

#img_to_face(path)

def extractSkin(image):
    # Taking a copy of the image
    img = image.copy()
    # Converting from BGR Colours Space to HSV
    img = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)

    # Defining HSV Threadholds
#     lower_threshold = np.array([0, 0, 0], dtype=np.uint8) # 0 48 80
#     upper_threshold = np.array([255, 255, 255], dtype=np.uint8) # 26 255 255

    lower_threshold = np.array([0, 48, 80], dtype=np.uint8) # 0 48 80 0 133 77
    upper_threshold = np.array([20, 255, 255], dtype=np.uint8) # 26 255 255 255 173 127

    # Single Channel mask,denoting presence of colours in the about threshold
    skinMask = cv2.inRange(img, lower_threshold, upper_threshold)

    # Cleaning up mask using Gaussian Filter
    skinMask = cv2.GaussianBlur(skinMask, (3, 3), 0)

    # Extracting skin from the threshold mask
    skin = cv2.bitwise_and(img, img, mask=skinMask)

    # Return the Skin image
    return cv2.cvtColor(skin, cv2.COLOR_HSV2BGR)

def removeBlack(estimator_labels, estimator_cluster):

    # Check for black
    hasBlack = False

    # Get the total number of occurance for each color
    occurance_counter = Counter(estimator_labels)

    # Quick lambda function to compare to lists
    def compare(x, y): return Counter(x) == Counter(y)

    # Loop through the most common occuring color
    for x in occurance_counter.most_common(len(estimator_cluster)):

        # Quick List comprehension to convert each of RBG Numbers to int
        color = [int(i) for i in estimator_cluster[x[0]].tolist()]

        # Check if the color is [0,0,0] that if it is black
        if compare(color, [0, 0, 0]) == True:
            # delete the occurance
            del occurance_counter[x[0]]
            # remove the cluster
            hasBlack = True
            estimator_cluster = np.delete(estimator_cluster, x[0], 0)
            break

    return (occurance_counter, estimator_cluster, hasBlack)

def getColorInformation(estimator_labels, estimator_cluster, hasThresholding=False):

    # Variable to keep count of the occurance of each color predicted
    occurance_counter = None

    # Output list variable to return
    colorInformation = []

    # Check for Black
    hasBlack = False

    # If a mask has be applied, remove th black
    if hasThresholding == True:

        (occurance, cluster, black) = removeBlack(
            estimator_labels, estimator_cluster)
        occurance_counter = occurance
        estimator_cluster = cluster
        hasBlack = black

    else:
        occurance_counter = Counter(estimator_labels)

    # Get the total sum of all the predicted occurances
    list_occurance_counter = list(occurance_counter.values())
    totalOccurance = int(sum(list_occurance_counter))

    # Loop through all the predicted colors
    for x in occurance_counter.most_common(len(estimator_cluster)):

        index = (int(x[0]))

        # Quick fix for index out of bound when there is no threshold
        index = (index-1) if ((hasThresholding & hasBlack)
                              & (int(index) != 0)) else index

        # Get the color number into a list
        color = estimator_cluster[index].tolist()

        # Get the percentage of each color
        color_percentage = (int(x[1])/totalOccurance)

        # make the dictionay of the information
        colorInfo = {"cluster_index": index, "color": color,
                     "color_percentage": color_percentage}

        # Add the dictionary to the list
        colorInformation.append(colorInfo)

    return colorInformation

def extractDominantColor(image, number_of_colors= 20, hasThresholding=False):

    # Quick Fix Increase cluster counter to neglect the black(Read Article)
    if hasThresholding == True:
        number_of_colors += 1

    # Taking Copy of the image
    img = image.copy()

    # Convert Image into RGB Colours Space
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

    # Reshape Image
    img = img.reshape((img.shape[0]*img.shape[1]), 3)

    # Initiate KMeans Object
    estimator = KMeans(n_clusters=number_of_colors, random_state=0)

    # Fit the image
    estimator.fit(img)

    # Get Colour Information
    colorInformation = getColorInformation(
        estimator.labels_, estimator.cluster_centers_, hasThresholding)
    return colorInformation

def savepic(user_id):
    """save pictures """
    image = cv2.imread("media/output/face.png")
    plt.subplot(3, 1, 1)
    plt.imshow(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
    skin = extractSkin(image)
    plt.subplot(3, 1, 2)
    plt.imshow(cv2.cvtColor(skin, cv2.COLOR_BGR2RGB))
    dominantcolors = extractDominantColor(skin, hasThresholding=True)
    colour_bar = plotColorBar(dominantcolors)
    plt.subplot(3, 1, 3)
    plt.axis("off")
    plt.imshow(colour_bar)
    plt.savefig('media/output/colorbar' + user_id + '.png')
    plt.close()
    #fig = Figure()
    #ax = fig.subplots()
    #ax.imshow(cv2.cvtColor(image, cv2.COLOR_BGR2RGB), cv2.cvtColor(skin, cv2.COLOR_BGR2RGB),
    # colour_bar)
    #buf = BytesIO()
    #fig.savefig('color_bar', format="png")
    return

def plotColorBar(colorInformation):
    """ create a 500*100 color bar """
    color_bar = np.zeros((100, 500, 3), dtype="uint8")

    top_x = 0
    for x in colorInformation:
        bottom_x = top_x + (x["color_percentage"] * color_bar.shape[1])

        color = tuple(map(int, (x['color'])))

        cv2.rectangle(color_bar, (int(top_x), 0),
                      (int(bottom_x), color_bar.shape[0]), color, -1)
        top_x = bottom_x
    return color_bar

def tone_analysis(img_path, user_id):
    """ tone_analysis method """
    # pylint: disable=line-too-long,no-member,too-many-locals,no-self-use
    img_to_face(img_path)
    image = cv2.imread("media/output/face.png")
    savepic(user_id)
    image = imutils.resize(image, width=250)
    skin = extractSkin(image)
    dominantColors = extractDominantColor(skin, hasThresholding=True)
    max_color = [0,0,0]
    sel_color = [0,0,0]
    site1 = 0
    site2 = 1
    for i in range(20):
        lum = dominantColors[i].get('color')[0]*0.2126 + dominantColors[i].get('color')[1]*0.7152 + dominantColors[i].get('color')[2]*0.0722
        max_lum = max_color[0]*0.2126 + max_color[1]*0.7152 + max_color[2]*0.0722
        sel_lum = sel_color[0]*0.2126 + sel_color[1]*0.7152 + sel_color[2]*0.0722
        if max_lum < lum:
            sel_color[0] = max_color[0]
            sel_color[1] = max_color[1]
            sel_color[2] = max_color[2]
            max_color[0] = dominantColors[i].get('color')[0]
            max_color[1] = dominantColors[i].get('color')[1]
            max_color[2] = dominantColors[i].get('color')[2]
            site2 = site1
            site1 = i
        elif sel_lum < lum:
            sel_color[0] = dominantColors[i].get('color')[0]
            sel_color[1] = dominantColors[i].get('color')[1]
            sel_color[2] = dominantColors[i].get('color')[2]
            site2 = i
    return sel_color


import discord, json, requests, pymysql.cursors
from discord.ext import commands
from utils import rpc_module, mysql_module, parsing
import math

rpc = rpc_module.Rpc()
mysql = mysql_module.Mysql()

class Withdraw:
    def __init__(self, bot):
        self.bot = bot
        config = parsing.parse_json('config.json')
        self.currency_symbol = config["currency_symbol"]
        self.explorer_url = config["explorer_url"]

    @commands.command(pass_context=True)
    async def withdraw(self, ctx, address: str, amount: float):
        """Withdraw coins from your account to any MyMn address"""
        snowflake = ctx.message.author.id
        if amount <= 0.0:
            await self.bot.say("{} **:warning:You cannot withdraw <= 0!:warning:**".format(ctx.message.author.mention))
            return

        abs_amount = abs(amount)
        if math.log10(abs_amount) > 8:
            await self.bot.say(":warning:**Invalid amount!**:warning:")
            return

        mysql.check_for_user(snowflake)

        conf = rpc.validateaddress(address)
        if not conf["isvalid"]:
            await self.bot.say("{} **:warning:Invalid address!:warning:**".format(ctx.message.author.mention))
            return

        ownedByBot = False
        for address_info in rpc.listreceivedbyaddess(0, True):
            if address_info["address"] == address:
                ownedByBot = True
                break

        if ownedByBot:
            await self.bot.say("{} **:warning:You cannot withdraw to an address owned by this bot!:warning:** Please use tip instead!".format(ctx.message.author.mention))
            return

        balance = mysql.get_balance(snowflake, check_update=True)
        if float(balance) < amount:
            await self.bot.say("{} **:warning:You cannot withdraw more money than you have!:warning:**".format(ctx.message.author.mention))
            return

        txid = mysql.create_withdrawal(snowflake, address, amount)
        if txid is None:
            await self.bot.say("{} your withdraw failed despite having the necessary balance! Please contact the support team".format(ctx.message.author.mention))
        else:
            await self.bot.say("{} **Withdrew {} {}! :moneybag:**\nView the transaction here: http://{}{}".format(ctx.message.author.mention, str(amount), self.currency_symbol, self.explorer_url, txid))

def setup(bot):
    bot.add_cog(Withdraw(bot))


"""
Docstring for this application
"""
from flask import Flask, render_template
from opt_seeker import list_of_functions_per_module, read_fnc

functions_cache = {}
attributes_cache = {}

# FLASK APP initialization
app = Flask(__name__)

@app.before_first_request
def  read_in_functions():
    """
    read in opt directory with all functions and their attributes
    :return:
    functions_cache: dict - list of all functions keyed by their module
    attributes_cache: dict - list of header data keyed by (module,function)
    """
    global functions_cache, attributes_cache
    result_dict = list_of_functions_per_module(return_attributes=True)
    functions_cache, attributes_cache = result_dict['functions'], result_dict['attributes']

# ROUTES
@app.route('/')
@app.route('/functions')
def opt_overview():
    """
    render all modules and their associated functions
    """

    return render_template(
        'page-functions.html',
        title='Functions',
        functions=functions_cache,
        attributes=attributes_cache,
    )

@app.route('/functions/<string:module>/<string:fnc>')
def opt_function_header(module, fnc):
    """
    read in function specified by module and fnc
    :param module: name of the module
    :param fnc: name of the function
    :return:
    """
    return_dict = read_fnc(module, fnc, return_attributes=False)
    fnc_name_full = 'fnc_{0}.sqf'.format(fnc)

    return render_template(
        'page-function-header.html',
        title=fnc_name_full,
        module=module,
        function=fnc,
        functions=functions_cache,
        attributes=attributes_cache,
        header=return_dict['header'],
        content=return_dict['content']
    )

if __name__ == '__main__':
    app.run()


import json
import pickle

import jieba
import os
import re
from gensim.models import word2vec

jieba.load_userdict("dict/dict.txt.big.txt")
jieba.load_userdict("dict/NameDict_Ch_v2")
jieba.load_userdict("dict/ntusd-negative.txt")
jieba.load_userdict("dict/ntusd-positive.txt")
jieba.load_userdict("dict/鄉民擴充辭典.txt")

print("--- 測試斷詞 ---")
print('/'.join(jieba.cut('柯P是柯市長', HMM=False)))
print("--- 測試斷詞 end ---")

# stop word
stopword_path = "dict/stopwords.json"
stopword_set = set()
with open(stopword_path, 'r') as csv:
    j_list = json.load(csv)
    for word in j_list:
        stopword_set.add(word)
print("讀取stop words")

# 情緒字詞
p_word_path = "dict/ntusd-positive.txt"
p_word_set = set()
with open(p_word_path, 'r') as csv:
    for line in csv:
        p_word_set.add(line)

n_word_path = "dict/ntusd-negative.txt"
n_word_set = set()
with open(n_word_path, 'r') as csv:
    for line in csv:
        n_word_set.add(line)

p_n_word_set = p_word_set.union(n_word_set)

comma_tokenizer = lambda x: [y for y in jieba.cut(x, cut_all=False, HMM=True) if y not in stopword_set]
#comma_tokenizer = lambda x: [y for y in jieba.cut(x, cut_all=False, HMM=True) if y in p_n_word_set]
#comma_tokenizer = lambda x: jieba.cut(x, cut_all=False, HMM=True)
#comma_tokenizer = lambda x: [y for y in x]

def clear_doc(doc):
    # 去除網址和符號
    r2 = r'https://[a-zA-Z0-9.?/&=:]*'
    r = '[’!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~]+'
    r3 = r'\n'
    text = re.sub(r2, '', doc)
    text = re.sub(r, '', text)
    # text = re.sub(r3, ' ', text)
    return text

def stream_docs(path, label):
    x = []
    y = []
    with open(path, 'r') as csv:
        for line in csv:
            text = clear_doc(line)
            if len(text) == 0:
                continue
            x.append([i for i in comma_tokenizer(text)])
            y.append(label)
    return x, y

def get_docs_labels(doc_streams):
    docs, y = [], []
    for doc_stream in doc_streams:
        try:
            t_x, t_y = doc_stream
            docs.extend(t_x)
            y.extend(t_y)
        except Exception:
            return None, None
    return docs, y

doc_streams = [stream_docs(path='data/n/broken-heart.txt', label=0), stream_docs(path='data/p/adulation.txt', label=1),
               stream_docs(path='data/p/kindness.txt', label=1)]

documents = get_docs_labels(doc_streams)[0]

word2vec_model_path = "model/word2vec.bin"

if not os.path.exists(word2vec_model_path):
    model = word2vec.Word2Vec(documents, size=250)
    model.save(word2vec_model_path)
else:
    model = word2vec.Word2Vec.load(word2vec_model_path)

result = model.most_similar("傷心")
print(result)
#print(model.similarity("傷心", "的"))


#快手笔试 数学科学家 数据科学题目1
#求ROC曲线以及AUC的计算逻辑

import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

data=pd.read_csv('C:/Users/Administrator/Desktop/label_score.csv')

#需要的值
FPR=[]
TPR=[]

#计算之前对于data按照score进行排序
da=data.sort_values(by='score',ascending=False)
n=len(da)
for i in range(n):
    ui=0
    tp=0
    fp=0
    fn=0
    tn=0
    for j ,row in da.iterrows():
        if ui<=i:
            if row['label']==1:
                tp+=1
            else:
                fp+=1
        else:
            if row['label']==1:
                fn+=1
            else:
                tn+=1
        ui+=1
    TPR.append(tp/(tp+tn))
    FPR.append(fp/(fp+fn))
sns.pointplot(x=FPR,y=TPR)
plt.xticks(rotation=270)
plt.show()

#计算AUC
#AUC表示了正例样本被分为正例的概率大于负利
#因此需要统计正例大于负利的概率
da1=da[da['label']==1]
da2=da[da['label']==-1]
m1=len(da1)
m2=len(da2)
kk=0
for i,row1 in da1.iterrows():
    for j ,row2 in da2.iterrows() :
        if row1['score']>row2['score']:
            kk+=1
AUC=kk/(m1*m2)

#如果要使用sklearn的话，则需要预测出来的label
#使用的包有：
from sklearn.metrics import accuracy_score
from sklearn import metrics
from sklearn.metrics import confusion_matrix
#confusion_matrix(y_true, y_pred)
from sklearn.metrics import classification_report
from sklearn.metrics import roc_auc_score#计算Roc的值


import os, pygame, sys, main, random, screen_text
pygame.init()
pygame.mixer.init()
pygame.mouse.set_visible( False )
path = os.getenv('APPDATA')
Width = 634
Length = 358

def main_(main_number, DISPLAYSURF):
    mousePointer = pygame.image.load("img/Pointer/mouse1.png")
    mousePointerClicked = pygame.image.load("img/pointer/mouse2.png")
    blackground = pygame.image.load("img/main_background.png")
    if main_number == 1:
        background = []
        background_range = 20
        read = 1
        for i in range(background_range):
            background.append(pygame.image.load("img/main_background"+str(read)+".png"))
            read += 1
    elif main_number == 2:
        background = [0, 1, 2, 3, 4]
        background[0] = pygame.image.load("img/drawables-new/main_background1.png")
        background[1] = pygame.image.load("img/drawables-new/main_background2.png")
        background[2] = pygame.image.load("img/drawables-new/main_background3.png")
        background[3] = pygame.image.load("img/drawables-new/main_background4.png")
        background[4] = pygame.image.load("img/drawables-new/main_background5.png")
        background_range = 5
    else:
        background = []
        background_range = 4
        read = 1
        for i in range(background_range):
            background.append(pygame.image.load("img/drawables-event/main_background"+str(read)+".png"))
            read += 1
    bgnumber = 0

    width_wall = pygame.image.load("img/main_widthwall.png")
    width_wall = pygame.transform.scale(width_wall, (Width+5, Length+1))
    length_wallleft = pygame.image.load("img/main_lengthwall1.png")
    length_wallright = pygame.image.load("img/main_lengthwall2.png")
    main_backbutton = pygame.image.load("img/main_back.png")
    backbutton_edge1 = pygame.image.load("img/main_backbutton_edge1.png")
    backbutton_edge2 = pygame.image.load("img/main_backbutton_edge2.png")
    main_fightbutton = pygame.image.load("img/main_fightbutton.png")
    main_levelupbutton = pygame.image.load("img/main_levelupbutton.png")
    main_shopbutton = pygame.image.load("img/main_shopbutton.png")
    main_limitbutton = pygame.image.load("img/main_limitbutton.png")
    backbutton_edge = [backbutton_edge1, backbutton_edge2]
    levelupbutton_edge1 = pygame.image.load("img/main_levelupbutton_edge1.png")
    levelupbutton_edge2 = pygame.image.load("img/main_levelupbutton_edge2.png")
    levelupbutton_edge = [levelupbutton_edge1, levelupbutton_edge2]
    levelupbuttonX = 0
    levelupbuttonY = 0
    shopbutton_edge1 = pygame.image.load("img/main_levelupbutton_edge1.png")
    shopbutton_edge2 = pygame.image.load("img/main_levelupbutton_edge2.png")
    shopbutton_edge = [shopbutton_edge1, shopbutton_edge2]
    shopbuttonX = 0
    shopbuttonY = 0
    fightbutton_edge1 = pygame.image.load("img/main_fightbutton_edge1.png")
    fightbutton_edge2 = pygame.image.load("img/main_fightbutton_edge2.png")
    fightbutton_edge = [fightbutton_edge1, fightbutton_edge2]
    fightbuttonX = 0
    fightbuttonY = 0
    limitbutton_edge1 = pygame.image.load("img/main_levelupbutton_edge1.png")
    limitbutton_edge2 = pygame.image.load("img/main_levelupbutton_edge2.png")
    limitbutton_edge = [limitbutton_edge1, limitbutton_edge2]
    limitbuttonX = 0
    limitbuttonY = 0
    go_home = False
    go_levelup = False
    go_shop = False
    go_fight = False
    go_limit = False
    backbuttonX = 0
    backbuttonY = 0
    timing = 0
    mouseX = 0
    mouseY = 0
    main_bgm = pygame.mixer.Sound('bgm/snd002.ogg')
    main_bgm.play(-1)
    textfile = open(path+'/TBD/data/xp.txt', 'r')
    xp = textfile.read()
    textfile = open(path+'/TBD/data/dog_feed.txt', 'r')
    dog_feed = textfile.read()
    if main_number == 1:
        clear_stagesstringtf = ""
    elif main_number == 2:
        clear_stagesstringtf = "2"
    else:
        clear_stagesstringtf = "3"
    textfile = open(path+'/TBD/data/clear_stages'+clear_stagesstringtf+'.txt', 'r')
    clear_stages = textfile.read()
    clear_stages = int(clear_stages)
    textfile.close()

    direction = ['left', False, 0]
    stagenumber = clear_stages+1
    bgX = 0
    if clear_stages > 0:
        for i in range(clear_stages):
            bgX += 634

    dog = [0, 1, 2]
    dog[0] = pygame.image.load("img/character/no1/no1_mm1white.png")
    dog[1] = pygame.image.load("img/character/no1/no1_mm2white.png")
    dog[2] = pygame.image.load("img/character/no1/no1_mm3white.png")
    number = 0

    fight_failed = [False, 0]
    fightfailedimage = pygame.image.load("img/fightfailedimage.png")

    edgenumber = 1
    backgroundedge = [0, 1]
    backgroundedge[0] = pygame.image.load("img/drawables-stages/button_edge1.png")
    backgroundedge[1] = pygame.image.load("img/drawables-stages/button_edge2.png")
    while True:
        mouseClicked = False
        for event in pygame.event.get():
            if event.type == pygame.QUIT and not go_home:
                go_home = True
                pygame.mixer.music.load("bgm/snd011.ogg")
                pygame.mixer.music.play(1)
                timing = 0
            elif event.type == pygame.MOUSEMOTION:
                mouseX, mouseY = event.pos
            elif event.type == pygame.MOUSEBUTTONUP:
                mouseX, mouseY = event.pos
                mouseClicked = True
        if mouseClicked:
            if mouseX < 301 and not stagenumber == clear_stages+1 and not direction[1]:
                if not (47 < mouseY < 86 and 13 < mouseX < 225):
                    direction = ['left', True, 0]
                    stagenumber += 1
            elif mouseX > 399 and not stagenumber == 1 and not direction[1]:
                direction = ['right', True, 0]
                stagenumber -= 1
        DISPLAYSURF.blit(blackground, (0, 0))
        bgplusX = 0
        bgnumber = 0
        for i in range(background_range):
            DISPLAYSURF.blit(background[bgnumber], (bgX-bgplusX, 0))
            bgplusX += 634
            bgnumber += 1
        if edgenumber == 0:
            edgenumber = 1
        elif edgenumber == 1:
            edgenumber = 0
        DISPLAYSURF.blit(backgroundedge[edgenumber], (0, 0))
        if number == 0:
            number = 1
        elif number == 1:
            number = 2
        elif number ==2:
            number = 0
        DISPLAYSURF.blit(dog[number], (0, 0))
        DISPLAYSURF.blit(main_fightbutton, (fightbuttonX, fightbuttonY))
        if main_number == 1:
            DISPLAYSURF.blit(main_levelupbutton, (levelupbuttonX, levelupbuttonY))
            DISPLAYSURF.blit(main_shopbutton, (shopbuttonX, shopbuttonY+50))
        if main_number == 2 and clear_stages > 2:
            DISPLAYSURF.blit(main_limitbutton, (limitbuttonX, limitbuttonY))
        DISPLAYSURF.blit(width_wall, (0, 0))
        DISPLAYSURF.blit(main_backbutton, (backbuttonX, backbuttonY))
        xp = str(xp)
        dog_feed = str(dog_feed)
        screen_text.item_text(xp, dog_feed, DISPLAYSURF)
        if direction[1]:
            if direction[2] == 0:
                direction[2] = 1
                if direction[0] == 'left':
                    bgX += 317
                elif direction[0] == 'right':
                    bgX -= 317
            elif direction[2] == 1:
                direction[2] = 2
                if direction[0] == 'left':
                    bgX += 217
                elif direction[0] == 'right':
                    bgX -= 217
            elif direction[2] == 2:
                direction[2] = 3
                if direction[0] == 'left':
                    bgX += 50
                elif direction[0] == 'right':
                    bgX -= 50
            elif direction[2] == 3:
                direction[2] = 4
                if direction[0] == 'left':
                    bgX += 30
                elif direction[0] == 'right':
                    bgX -= 30
            elif direction[2] == 4:
                direction[2] = 0
                direction[1] = False
                if direction[0] == 'left':
                    bgX += 20
                elif direction[0] == 'right':
                    bgX -= 20
        if fight_failed[0]:
            if fight_failed[1] < 31:
                fight_failed[1] += 1
                DISPLAYSURF.blit(fightfailedimage, (0, 0))
        if 4 < mouseX < 59 and 305 < mouseY < 356 and not go_home: #twinkling
            pygame.mixer.music.load("bgm/snd010.ogg")
            pygame.mixer.music.play(1)
            random_color = random.randint(0, 1)
            DISPLAYSURF.blit(backbutton_edge[random_color], (0, 10))
        if main_number == 1:
            if 147 < mouseY < 186 and 13 < mouseX < 225 and not go_shop: #
                pygame.mixer.music.load("bgm/snd010.ogg")
                pygame.mixer.music.play(1)
                random_color = random.randint(0, 1)
                DISPLAYSURF.blit(shopbutton_edge[random_color], (0, 50))
            if 97 < mouseY < 136 and 13 < mouseX < 225 and not go_levelup: #twinkling2
                pygame.mixer.music.load("bgm/snd010.ogg")
                pygame.mixer.music.play(1)
                random_color = random.randint(0, 1)
                DISPLAYSURF.blit(levelupbutton_edge[random_color], (0, 0))
        if main_number == 2 and clear_stages > 2:
            if 97 < mouseY < 136 and 13 < mouseX < 225 and not go_limit: #
                pygame.mixer.music.load("bgm/snd010.ogg")
                pygame.mixer.music.play(1)
                random_color = random.randint(0, 1)
                DISPLAYSURF.blit(limitbutton_edge[random_color], (0, 0))
        if 47 < mouseY < 86 and 13 < mouseX < 225 and not go_fight: #twinkling3
            pygame.mixer.music.load("bgm/snd010.ogg")
            pygame.mixer.music.play(1)
            random_color = random.randint(0, 1)
            DISPLAYSURF.blit(fightbutton_edge[random_color], (0, 0))
        if mouseClicked:
            if main_number == 1:
                if 147 < mouseY < 186 and 13 < mouseX < 225 and not go_shop: #Go Shop
                    pygame.mixer.music.load("bgm/snd011.ogg")
                    pygame.mixer.music.play(1)
                    timing = 0
                    main_bgm.stop()
                    go_shop = True
                if 97 < mouseY < 136 and 13 < mouseX < 225 and not go_levelup: #Go powerup
                    pygame.mixer.music.load("bgm/snd011.ogg")
                    pygame.mixer.music.play(1)
                    timing = 0
                    main_bgm.stop()
                    go_levelup = True
            if main_number == 2 and clear_stages > 2:
                if 97 < mouseY < 136 and 13 < mouseX < 225 and not go_limit: #Go limit
                    pygame.mixer.music.load("bgm/snd011.ogg")
                    pygame.mixer.music.play(1)
                    timing = 0
                    main_bgm.stop()
                    go_limit = True
            if 47 < mouseY < 86 and 13 < mouseX < 225 and not go_fight: #Go Fight
                xp = int(xp)
                if main_number == 1:
                    if stagenumber <= clear_stages+1 and not stagenumber == 20:
                        xp -= 100
                        pygame.mixer.music.load("bgm/snd011.ogg")
                        pygame.mixer.music.play(1)
                        timing = 0
                        main_bgm.stop()
                        go_fight = True
                    else:
                        pygame.mixer.music.load("bgm/snd015.ogg")
                        pygame.mixer.music.play(1)
                        fight_failed[0] = True
                        fight_failed[1] = 0
                elif main_number == 2:
                    if stagenumber < clear_stages+2 and not stagenumber == 6:
                        xp -= 100
                        pygame.mixer.music.load("bgm/snd011.ogg")
                        pygame.mixer.music.play(1)
                        timing = 0
                        main_bgm.stop()
                        go_fight = True
                    else:
                        pygame.mixer.music.load("bgm/snd015.ogg")
                        pygame.mixer.music.play(1)
                        fight_failed[0] = True
                        fight_failed[1] = 0
                else:
                    if stagenumber == 4:
                        xp -= 100
                        pygame.mixer.music.load("bgm/snd011.ogg")
                        pygame.mixer.music.play(1)
                        timing = 0
                        main_bgm.stop()
                        go_fight = True
                    else:
                        pygame.mixer.music.load("bgm/snd015.ogg")
                        pygame.mixer.music.play(1)
                        fight_failed[0] = True
                        fight_failed[1] = 0
                xp = str(xp)
                textfile = open(path+"/TBD/data/xp.txt", "w")
                textfile.write(xp)
                textfile.close()
            if 4 < mouseX < 59 and 305 < mouseY < 356 and not go_home: #Back
                go_home = True
                pygame.mixer.music.load("bgm/snd011.ogg")
                pygame.mixer.music.play(1)
                timing = 0
        if go_shop:
            if timing == 1:
                main_shopbutton = pygame.image.load("img/main_shopbutton.png")
                main_shopbutton = pygame.transform.scale(main_shopbutton, (Width+15, Length+15))
                shopbuttonX = 0
                shopbuttonY = -10
            if timing == 2:
                main_shopbutton = pygame.image.load("img/main_shopbutton.png")
                main_shopbutton = pygame.transform.scale(main_shopbutton, (Width-50, Length-50))
                shopbuttonY = 15
                shopbuttonX = 10
            if timing == 3:
                main_shopbutton = pygame.image.load("img/main_shopbutton.png")
                main_shopbutton = pygame.transform.scale(main_shopbutton, (Width, Length))
                shopbuttonX = 0
                shopbuttonY = 0
            if timing == 4:
                main.close_door('shop', 0, "0.0.0.0", 0)
                go_shop = False
                break
            timing += 1
        if go_levelup:
            if timing == 1:
                main_levelupbutton = pygame.image.load("img/main_levelupbutton.png")
                main_levelupbutton = pygame.transform.scale(main_levelupbutton, (Width+15, Length+15))
                levelupbuttonX = 0
                levelupbuttonY = -10
            if timing == 2:
                main_levelupbutton = pygame.image.load("img/main_levelupbutton.png")
                main_levelupbutton = pygame.transform.scale(main_levelupbutton, (Width-50, Length-50))
                levelupbuttonY = 15
                levelupbuttonX = 10
            if timing == 3:
                main_levelupbutton = pygame.image.load("img/main_levelupbutton.png")
                main_levelupbutton = pygame.transform.scale(main_levelupbutton, (Width, Length))
                levelupbuttonX = 0
                levelupbuttonY = 0
            if timing == 4:
                main.close_door('powerup', 0, "0.0.0.0", 0)
                go_levelup = False
                break
            timing += 1
        if go_limit:
            if timing == 1:
                main_limitbutton = pygame.image.load("img/main_limitbutton.png")
                main_limitbutton = pygame.transform.scale(main_limitbutton, (Width+15, Length+15))
                limitbuttonX = 0
                limitbuttonY = -10
            if timing == 2:
                main_limitbutton = pygame.image.load("img/main_limitbutton.png")
                main_limitbutton = pygame.transform.scale(main_limitbutton, (Width-50, Length-50))
                limitbuttonY = 15
                limitbuttonX = 10
            if timing == 3:
                main_limitbutton = pygame.image.load("img/main_limitbutton.png")
                main_limitbutton = pygame.transform.scale(main_limitbutton, (Width, Length))
                limitbuttonX = 0
                limitbuttonY = 0
            if timing == 4:
                main.close_door('limit', 0, "0.0.0.0", 0)
                go_limit = False
                break
            timing += 1
        if go_fight:
            if timing == 1:
                main_fightbutton = pygame.image.load("img/main_fightbutton.png")
                main_fightbutton = pygame.transform.scale(main_fightbutton, (Width+15, Length+15))
                fightbuttonX = 0
                fightbuttonY = -10
            if timing == 2:
                main_fightbutton = pygame.image.load("img/main_fightbutton.png")
                main_fightbutton = pygame.transform.scale(main_fightbutton, (Width-50, Length-50))
                fightbuttonY = 15
                fightbuttonX = 10
            if timing == 3:
                main_fightbutton = pygame.image.load("img/main_fightbutton.png")
                main_fightbutton = pygame.transform.scale(main_fightbutton, (Width, Length))
                fightbuttonX = 0
                fightbuttonY = 0
            if timing == 4:
                if main_number == 1:
                    main.close_door('fight', stagenumber, "0.0.0.0", 0)
                elif main_number == 2:
                    main.close_door('fight', -stagenumber, "0.0.0.0", 0)
                else:
                    main.close_door('fight', str(stagenumber), "0.0.0.0", 0)
                go_fight = False
                break
            timing += 1
        elif go_home: #main go to start screen motion
            if timing == 1:
                main_backbutton = pygame.transform.scale(main_backbutton, (Width+15, Length+15))
                backbuttonX = 0
                backbuttonY = -10
            if timing == 2:
                main_backbutton = pygame.image.load("img/main_back.png")
                main_backbutton = pygame.transform.scale(main_backbutton, (Width-50, Length-50))
                backbuttonY = 45
                backbuttonX = 3
            if timing == 3:
                main_backbutton = pygame.image.load("img/main_back.png")
                backbuttonX = 0
                backbuttonY = 0
            if timing == 4:
                go_home = False
                main_bgm.stop()
                main.FadeOut('start_screen')
                break
            timing += 1
        if mouseClicked:
            DISPLAYSURF.blit(mousePointerClicked, (mouseX-10, mouseY-10))
        else:
            DISPLAYSURF.blit(mousePointer, (mouseX-10, mouseY-10))
        pygame.time.wait(10)
        pygame.display.update()


import os
import glob
import argparse
import re, string, unicodedata
import numpy as np
import pandas as pd
import nltk
import contractions
import inflect
from nltk import word_tokenize
from nltk.corpus import stopwords, wordnet
from nltk.stem import LancasterStemmer, WordNetLemmatizer

def splitter(text):
    return " ".join(text.split("|"))

# Detaching concatenated words
def detach_words(text):
    """Detach Concatenated Words"""
    idx = 0
    new_text = []
    for i in text:
        idx+=1
        if i.isupper():
            if not text[idx-1] == '-':# or not text[idx-3] == ' - ':
                new_text.append(' ' + i)
            else:
                new_text.append(i)
    new_text = re.sub("w/","", re.sub("^ ","", re.sub("  "," ", "".join(new_text))))
    new_text = re.sub(" -","-", re.sub("- ","-", re.sub(" - ","-", new_text)))
    new_text = re.sub("^/","", re.sub("^-","", new_text))
    return new_text

# Replace contractions
def replace_contractions(text):
    """Replace contractions in string of text"""
    try:
        x = contractions.fix(text)
    except UnicodeEncodeError as e:
        return ""
    return x

# Tokenize
def tokenize_text(text):
    try:
        x = nltk.word_tokenize(text)
    except UnicodeEncodeError as e:
        return ""
    return x

# Remove non ascii characters
def remove_non_ascii(words):
    """Remove non-ASCII characters from list of tokenized words"""
    new_words = []
    for word in words:
        word = re.sub("[ ]"," ", re.sub("^/","", re.sub("^-","",word)))
        new_word = unicodedata.normalize('NFKD', word).encode('ascii', 'ignore').decode('utf-8', 'ignore')
        new_words.append(new_word)
    return new_words

# Convert every word to lowercase
def to_lowercase(words):
    """Convert all characters to lowercase from list of tokenized words"""
    new_words = []
    for word in words:
        new_word = re.split('([a-zA-Z]+)/', word.lower())
        new_words.extend(new_word)
    return new_words

# Remove punctuations
def remove_punctuation(words):
    """Remove punctuation from list of tokenized words"""
    new_words = []
    for word in words:
        new_word = re.sub(r'\.',' ', re.sub(r'[^\./\w\s-]', '', word))
        if new_word != '':
            new_words.append(new_word)
    return new_words

# Replace numbers
def replace_numbers(words):
    """Replace all interger occurrences in list of tokenized words with textual representation"""
    p = inflect.engine()
    new_words = []
    for word in words:
        if word.isdigit():
            new_word = p.number_to_words(word)
            new_words.append(new_word)
        else:
            new_words.append(word)
    return new_words

# Remove stopwords
def remove_stopwords(words):
    """Remove stop words from list of tokenized words"""
    new_words = []
    for word in words:
        if word not in stopwords.words('english'):
            new_words.append(word)
    return new_words

def remove_single_letters(words):
    """Removes single letter words"""
    new_words = []
    for i in words:
        if len(i) > 2:
            new_words.append(i)
    return new_words

def remove_only_numbers(words):
    """Removes only numbers without any letters"""
    new_words = []
    for i in words:
        if re.search("\d", i):
            if re.search("[a-zA-Z]", i):
                new_words.append(i)
        else:
            new_words.append(i)
    return new_words

def pos_tagger(words):
    """Returns only nouns and adjectives"""
    new_words = []
    for word,tag in nltk.pos_tag(words):
        if tag.startswith('J') or tag.startswith('N'):
            new_words.append(word)
    return sorted(set(new_words))

def stem_words(words):
    """Stem words in list of tokenized words"""
    stemmer = LancasterStemmer()
    stems = []
    for word in words:
        stem = stemmer.stem(word)
        stems.append(stem)
    return stems

def convert_numbers_to_words(text):
    """Breaks alphanumeric words to counterparts"""
    text = re.sub(r"\W+", " ", text)
    def number_inflection(text):
        idx = 0
        new_text = []
        try:
            for i in text:
                if i.isdigit():
                    if not text[idx+1].isdigit() and not text[idx+1].isspace():
                        new_text.extend([i,' '])
                    else:
                        new_text.append(i)
                else:
                    new_text.append(i)
                idx+=1
        except IndexError as e:
            new_text.append(i)
            pass
        return new_text
    return ''.join(number_inflection(text))

def get_wordnet_pos(words):
    for word in words:
        if word.startswith('J'):
            return wordnet.ADJ
        elif word.startswith('V'):
            return wordnet.VERB
        elif word.startswith('N'):
            return wordnet.NOUN
        elif word.startswith('R'):
            return wordnet.ADV
        else:
            return wordnet.NOUN

def lemmatize(words):
    """Lemmatize words in the list of tokenized words"""
    lemma = []
    lemmatizer = WordNetLemmatizer()
    for word, tag in nltk.pos_tag(words):
        tag = get_wordnet_pos(tag)
        lemma.append(lemmatizer.lemmatize(word, tag))
    return lemma

def get_word_frequency_from_pd(text, remove_stopwords_bool=False):
#     try:
#         text = " ".join(str(text).split('|')[2:])
#     except UnicodeEncodeError as e:
#         pass
    text = re.sub(r"#(\w+)", "", str(text).lower())
    text = re.sub(r'\|',' ', str(text))
    text = replace_contractions(text)
    text = convert_numbers_to_words(text)
    words = tokenize_text(text)
    words = replace_numbers(words)
    words = remove_non_ascii(words)
    words = to_lowercase(words)
    words = remove_punctuation(words)
    words = remove_only_numbers(words)
    words = lemmatize(words)
    if remove_stopwords_bool:
        words = remove_stopwords(words)
    text = re.sub(r" +"," ", re.sub("[^\w\s]"," ", re.sub("[0-9]+","", " ".join(words))))
    #words = pos_tagger(words)
    #words = remove_single_letters(words)
    # if remove_similar:
    #     words, master_list = remove_similar_words(words, master_list)
    return text

def get_word_frequency_from_pn(text, remove_stopwords_bool=False):
    text = re.sub(r"#(\w+)", "", str(text).lower())
    text = re.sub(r'\|',' ', str(text))
    text = replace_contractions(text)
    text = convert_numbers_to_words(text)
    words = tokenize_text(text)
    words = replace_numbers(words)
    words = remove_non_ascii(words)
    words = to_lowercase(words)
    words = remove_punctuation(words)
    words = remove_only_numbers(words)
    words = lemmatize(words)
    if remove_stopwords_bool:
        words = remove_stopwords(words)
    text = re.sub(r" +"," ", re.sub("[^\w\s]"," ", re.sub("[0-9]+","", " ".join(words))))
    return text


import pandas as pd
from sklearn.metrics import confusion_matrix
from sklearn.tree import DecisionTreeClassifier

if __name__ == "__main__":
    train_data = pd.read_csv("train.csv", sep=',', header=0)
    train_features = train_data.values[:, 0:8]
    train_outcome = train_data.values[:, 8]
    decision_tree = DecisionTreeClassifier(criterion="entropy", min_samples_split=9, class_weight={1: 4, 0: 1})
    decision_tree.fit(train_features, train_outcome)
    test_data = pd.read_csv("test.csv", sep=',', header=0)
    test_features = test_data.values[:, 0:8]
    test_real_outcome = test_data.values[:, 8]
    test_outcome = decision_tree.predict(test_features)
    print(confusion_matrix(test_real_outcome, test_outcome))


import os
from itertools import takewhile
import subprocess

import ranger.api
import ranger.core.runner
from ranger.api.commands import Command

old_hook_init = ranger.api.hook_init

def fullpath(path):
    return os.path.normpath(os.path.abspath(os.path.expanduser(path)))

def splitpath(path):
    """
    Split path into components, discard empty ones (produced by double slashes
    for instance.
    """
    return list(filter(None, directory(path).split(os.path.sep)))

def directory(path):
    """
    Returns a directory name of the supplied path, in case it's not directory
    already.
    """
    if os.path.exists(path) and not os.path.isdir(path):
        path = os.path.dirname(path)
    return path

def dirs(path):
    """
    List directories in supplied path
    """
    path = directory(path)
    return (f for f in os.listdir(path)
            if os.path.isdir(os.path.join(path, f)))

def allnamesequal(name):
    return all(n == name[0] for n in name[1:])

def commonprefix(paths, sep=os.path.sep):
    """
    Due to a bug in os.path.commonprefix implementation, we use this instead
    """
    bydirectorylevels = zip(*[p.split(sep) for p in paths])
    return sep.join(x[0] for x in takewhile(allnamesequal, bydirectorylevels))

def find_subpath(src, dst):
    srcdir, substring = os.path.split(src)
    common_prefix = commonprefix([src, dst])
    srcdir = srcdir[len(common_prefix):]
    if dst in src:
        # search left to right
        # skip the very first directory (otherwise pointless)
        searchlist = splitpath(srcdir)[1:]
        ltr = True
    else:
        # search right to left
        searchlist = reversed(splitpath(srcdir))
        ltr = False
    for destdir in searchlist:
        if destdir in dirs(dst):
            if ltr:
                pos = src[src.find(destdir):]
                substring = src[pos:]
                break
            else:
                pos = src.rfind(destdir)
                substring = src[pos:]
    return substring

class paste_subtree(Command):
    """
    :paste_subree

    Paste from copy_buffer to current directory. Try to find the common subpath
    and preserve as much of the original directory structure as possible
    """

    def reload_dir(self, dst):
        dst = self.fm.get_directory(dst)
        dst.unload()
        dst.load_content()

    def execute(self):
        dst = str(self.fm.thisdir)
        try:
            for src in (str(p) for p in self.fm.copy_buffer):
                paste_subtree_call(self.fm, src, dst)
                self.reload_dir(dst)
        finally:
            self.fm.copy_buffer.clear()
            self.fm.cut_buffer = False

def paste_subtree_call(fm, src, dst):
    """
    # cd dstdir
    # rsync -Ravz a/b/c/d.ext /home/username/Documents
    # rsync ^args ^relpath    ^dstdir
    # cd -
    """

    src = fullpath(src)
    if not os.path.exists(src):
        raise OSError('[ERROR] not found: {}'.format(src))

    subpath = find_subpath(src, dst)
    chdir = src[:-len(subpath)]

    # # DEBUG:
    result = os.path.join(dst, subpath)
    fm.notify(' chdir: {}'.format(chdir))
    fm.notify('subpth: {}{}'.format(' ' * (len(chdir)), subpath))
    fm.notify('   src: {}'.format(src))
    fm.notify('   dst: {}'.format(dst))
    fm.notify('result: {}'.format(result))

    # FIXME: this fails with ' i paths
    command = "cd '{chdir}' && rsync -Ravz '{subpath}' '{dst}' ; cd -".format(
        chdir=chdir,
        subpath=subpath,
        dst=dst)
    fm.notify(command)
    proc = fm.execute_command(command, stdout=subprocess.PIPE)
    stdout, stderr = proc.communicate()
    if stdout:
        for line in reversed(stdout.decode().split('\n')):
            fm.notify('rsync: {}'.format(line))
    if stderr:
        fm.notify(stderr.decode(), bad=True)

def hook_init(fm):
    fm.commands.commands['paste_subtree'] = paste_subtree
    return old_hook_init(fm)

ranger.api.hook_init = hook_init


# -*- coding: utf-8 -*-
# @Time    : 2019/7/17 12:32
# @Author  : Valette
# @Email   : foxmail
# @File    : present.py
# @Software: PyCharm

# 模块功能：
# 数据可视化
import numpy as np
import json
from pyecharts.charts import BMap
from pyecharts import options as opts
from pyecharts.charts import Geo

from mysql_connection import cur, conn
from Config import json_file, BAIDU_AK

def get_pie_type(sql_price_data, json_file) -> BMap:
    c = (
        BMap()
        .add_schema(
            baidu_ak=BAIDU_AK,
            center=[120.1222120000,36.0031470000],
            zoom = 12
        )
        .add_coordinate_json(json_file)
        .add(
            "在售楼盘",
            [[name, int(price)] for name, price in sql_price_data],
            label_opts=opts.LabelOpts(formatter="{b}"),
        )
        .set_series_opts(label_opts=opts.LabelOpts(is_show=False))
        .set_global_opts(title_opts=opts.TitleOpts(title="黄岛楼盘地理分布"))
    )
    return c

def get_pie_type() -> Pie:
    '''
        楼盘类型 饼状图
        :return:
        '''
    sql = "SELECT `类型` FROM `楼盘`"
    cur.execute(sql)
    data = cur.fetchall()
    x = {}
    for i in data:
        if i[0] is not None:
            if i[0] in x:
                x[i[0]] += 1
            else:
                x[i[0]] = 1

    c = (
        Pie()
            .add("", [(k,x[k]) for k in x.keys()])
            .set_global_opts(title_opts=opts.TitleOpts(title="Pie-楼盘类型"))
            .set_series_opts(label_opts=opts.LabelOpts(formatter="{b}: {c}"))
    )
    return c

def geo_distribution():
    cur.execute("SELECT `楼盘名称`, `参考单价`FROM `楼盘` WHERE `坐标` IS NOT NULL AND `参考单价` IS NOT NULL")
    sql_price_data = cur.fetchall()
    map = get_bmap_geo(sql_price_data, json_file)
    map.render("outputs/地理分布.html")

def type_distribution():
    diagram = get_pie_type()
    diagram.render("outputs/楼盘类型分布.html")

if __name__ == '__main__':
    # price_tendence()
    # type_pie()
    geo_distribution()
    type_distribution()
    # cur.close()
    # conn.close()


import datetime
""" from rest_framework.serializers import ValidationError
from .utils import * """

class ValidationUtils:
    @staticmethod
    def list_duplicates(seq):
        seen = set()
        seen_add = seen.add
        # adds all elements it doesn't know yet to seen and
        # all other to seen_twice
        seen_twice = set(x for x in seq if x in seen or seen_add(x))
        # turn the set into a list (as requested)
        return list(seen_twice)

    @staticmethod
    def Find_Duplicates(fields_data, field_name, strType, usesForeignKey):
        strCombinedErrorMessage = None
        field_names = []
        for field_data in fields_data:
            str_data = field_data[field_name]
            if usesForeignKey is True:
                field_names.append(str_data.id)
            else:
                field_names.append(str_data)
        field_dupes = ValidationUtils.list_duplicates(field_names)
        if field_dupes is not None:
            if len(field_dupes) > 0:
                strErrorMessage = '[' + ','.join(map(str, field_dupes)) + '].'
                strCombinedErrorMessage = "Following duplicate " + field_name \
                                          + " found " + strErrorMessage + \
                                          " Duplicate " + field_name + \
                                          " not allowed within " + strType
        return strCombinedErrorMessage

    @staticmethod
    def IsValidField(oRiskFieldData):
        strErrorMessage = None

        # Check if
        if(oRiskFieldData is None):
            strErrorMessage = "riskfield is not found. Post valid riskfield"
        # print("In ValidationUtil IsValidField")
        # print(oRiskFieldData)
        if oRiskFieldData.risk_type_field_enum == "integer":
            bIsInteger = ValidationUtils.isanumber(
                        oRiskFieldData.risk_field_value)
            if bIsInteger is False:
                strErrorMessage = oRiskFieldData.risk_type_field_name + \
                                  " configured as " + \
                                  oRiskFieldData.risk_type_field_enum + \
                                  " This field must be valid integer."

        elif (oRiskFieldData.risk_type_field_enum == "number" or
              oRiskFieldData.risk_type_field_enum == "currency"):
            bIsNumber = ValidationUtils.isanumber(
                         oRiskFieldData.risk_field_value)
            if bIsNumber is False:
                strErrorMessage = oRiskFieldData.risk_type_field_name + \
                                 " configured as " + \
                                 oRiskFieldData.risk_type_field_enum + \
                                 " This field must be an real number."

        elif oRiskFieldData.risk_type_field_enum == "date":
            bIsDate = ValidationUtils.isadate(
                        oRiskFieldData.risk_field_value)
            if bIsDate is False:
                strErrorMessage = oRiskFieldData.risk_type_field_name + \
                 " configured as " + \
                 oRiskFieldData.risk_type_field_enum + \
                 " Enter date in MM/dd/yyyy format"

        elif oRiskFieldData.risk_type_field_enum == "string":
            bIsString = ValidationUtils.isastring(
                          oRiskFieldData.risk_field_value)
            if bIsString is False:
                strErrorMessage = oRiskFieldData.risk_type_field_name + \
                  " configured as " + \
                  oRiskFieldData.risk_type_field_enum + \
                  " Enter valid string"
        # print(strErrorMessage)
        return strErrorMessage

    @staticmethod
    def isanumber(inp):
        try:
            val = int(inp)
            return True
        except ValueError:
            try:
                val = float(inp)
                return True
            except ValueError:
                return False

    @staticmethod
    def isadate(inp):
        try:
            month, day, year = inp.split('/')
            isValidDate = True
            datetime.datetime(int(year), int(month), int(day))
        except ValueError:
            isValidDate = False
        return isValidDate

    @staticmethod
    def isastring(inp):
        isValidString = bool(inp and inp.strip())
        return isValidString


import plotly.figure_factory as ff
import plotly.graph_objects as go
import statistics
import random
import pandas as pd
import csv

df=pd.read_csv("D:/C-111/data1.csv")
data=df["Math_score"].tolist()

#Plotting the Graph
#fig=ff.create_distplot([data],["Math Scores"],show_hist=False)
#fig.show()

#Calculating the Mean and Standard Deviation of The Population Data
mean=statistics.mean(data)
std_deviation=statistics.stdev(data)

print("Mean of Population --> ",mean)
print("Standard Devaition of Population --> ",std_deviation)

def random_set_of_mean(counter):
    dataset=[]
    for i in range(0,counter):
        random_index=random.randint(0,len(data)-1)
        value=data[random_index]
        dataset.append(value)
    mean=statistics.mean(dataset)
    return mean

#Pass the number of time you want the mean of data points as a parameter in range function in for loop
mean_list=[]
for i in range(0,1000):
    set_of_means=random_set_of_mean(100)
    mean_list.append(set_of_means)

#Calculating the mean and std_deviation of the sampling distribution
std_deviation=statistics.stdev(mean_list)
mean=statistics.mean(mean_list)

print("Mean of sampling distribution --> ",mean)

#Plotting the mean of the sampling
fig=ff.create_distplot([mean_list],["student marks"],show_hist=False)
fig.add_trace(go.Scatter(x=[mean,mean],y=[0,0.20],mode="lines",name="MEAN"))

#Finding the standard deviaitions starting and ending value
first_std_deviation_start,first_std_deviation_end=mean-std_deviation,mean+std_deviation
second_std_deviation_start,second_std_deviation_end=mean-(2*std_deviation),mean+(2*std_deviation)
third_std_deviation_start,third_std_deviation_end=mean-(3*std_deviation),mean+(3*std_deviation)

print("Std 1",first_std_deviation_start,first_std_deviation_end)
print("Std 2",second_std_deviation_start,second_std_deviation_end)
print("Std 3",third_std_deviation_start,third_std_deviation_end)

#Plotting the graph with traces

fig.show()


# -*- coding:utf-8 -*-

from threading import Thread
import multiprocessing
import time
import subprocess
import sys

from function import console

class MinecraftServerProcess(multiprocessing.Process):
    ''' MinecraftServerProcess '''
    def __init__(self,servername,cwd,run_command):
        multiprocessing.Process.__init__(self)
        self.servername = servername
        self.console_queue = None
        self.subproc = None
        self.run_command = run_command
        self.console_code = 'gbk'
        self.subexit = False
        self.cwd = cwd

    def command_queue(self):
        '''
        服务器进程 通信获取
        通过这个来管理中心可以自由的压入任务到队列，其中的一个线程会进行处理
        '''
        self.command_queue = multiprocessing.Queue()
        return self.command_queue

    def set_cosole_queue(self,queue):
        self.console_queue = queue

    def _command_thread_loop(self):
        ''' 此线程负责读入命令 管理中心 发来的命令 '''
        while self.subexit == False:
            command = self.command_queue.get()
            if command['command'] == 'exec':
                subproc_command = command['msg'] + '\r\n'
                self.subproc.stdin.write(subproc_command.encode(self.console_code))
                self.subproc.stdin.flush()

            if command['command'] == 'exit':
                self.subexit = True
                break
        console.log(-1,'[',self.servername,'] 服务器 command_thread_loop 线程回收')

    def _console_thread_loop(self):
        ''' 此线程负责写入控制台到 管理中心 的队列 '''
        while self.subexit == False:
            #读取 Minecraft 子进程 stdout
            output = self.subproc.stdout.readline()
            output = output.decode(self.console_code)
            #output = output.strip('\n').strip('\r\n')
            if not output:
                self.subexit = True
                self.command_queue.put({'command':'exit'})
                break
            if output != '' or len(output) > 1:
                self.console_queue.put({
                    'servername' : self.servername,
                    'command'    : 'console',
                    'msg'        : output
                })

        console.log(-1,'[',self.servername,'] 服务器 console_thread_loop 线程回收')

    def run(self):
        console.log(-1,'[',self.servername,'] 服务器 MinecraftServerProcess 进程开启')
        #创建命令读取线程
        self.subproc = subprocess.Popen(
            self.run_command,
            cwd = self.cwd,
            stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr = subprocess.PIPE,
            shell=True
            )
        self._command_thread_loop = Thread(target=self._command_thread_loop)
        self._console_thread_loop = Thread(target=self._console_thread_loop)
        self._command_thread_loop.start()
        self._console_thread_loop.start()
        self._console_thread_loop.join()
        self._command_thread_loop.join()
        self._server_exit_event()
        console.log(-1,'[',self.servername,'] 服务器 MinecraftServerProcess 进程回收')
        # exit(0)

    def _server_exit_event(self):
        self.console_queue.put({
                    'servername' : self.servername,
                    'command'    : 'close',
                })

class MinecraftServer(MinecraftServerProcess):
    ''' MinecraftServerProcess '''
    def __init__(self,servername,cwd,run_command,premssion_dict={}):
        super().__init__(servername,cwd,run_command)
        self.start_time = time.time()
        self.start_time_str = time.strftime('%Y-%m-%d %H:%M:%I',time.localtime(self.start_time))
        self._permission = premssion_dict

    def get_run_time(self):
        now = time.time()
        all_time = now - self.start_time
        return round(all_time)
        #return time.strftime('%Y-%m-%d %H:%M:%I',time.localtime(all_time))

    def get_start_time_str(self):
        return self.start_time_str

    def edit_user_premssion(self,username,premssion_dict):
        '''
        premssion_dict
        {'PlayerName':{
            'view' : True ,     #查看服务器界面权限
            'command' : True,   #向服务器发送命令权限
            'button' : True,      #控制服务器开关 开启 关闭 重启 强制 权限
        }}
        '''
        self._permission[username] = premssion_dict

    def get_info(self):
        ''' 获取这个 Minecraft 服务器的所有显示信息 '''
        return {
            'servername' : self.servername,
            'start_time' : self.start_time_str,
            'run_time'   : self.get_run_time(),
            'cwd'        : self.cwd,
            'subexit'    : self.subexit,
            'run_command': self.run_command,
            'permission' : self._permission
        }


#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
from ai_flow.util.json_utils import Jsonable
from typing import Text
from ai_flow.common.properties import Properties

class WorkflowMeta(Jsonable):
    """define workflow meta"""

    def __init__(self,
                 name: Text,
                 project_id: int,
                 properties: Properties = None,
                 create_time: int = None,
                 update_time: int = None,
                 uuid: int = None
                 ) -> None:
        """

        :param name: the workflow name
        :param project_id: the uuid of project which contains this workflow
        :param properties: the workflow properties
        :param create_time: create time represented as milliseconds since epoch.
        :param update_time: update time represented as milliseconds since epoch.
        :param uuid: uuid in database
        """

        self.name = name
        self.project_id = project_id
        self.properties = properties
        self.create_time = create_time
        self.update_time = update_time
        self.uuid = uuid

    def __str__(self):
        return '<\n' \
               'WorkflowMeta\n' \
               'uuid:{},\n' \
               'name:{},\n' \
               'project_id:{},\n' \
               'properties:{},\n' \
               'create_time:{},\n' \
               'update_time:{},\n' \
               '>'.format(self.uuid,
                          self.name,
                          self.project_id,
                          self.properties,
                          self.create_time,
                          self.update_time)

def create_workflow(name: Text,
                    project_id: Text,
                    properties: Properties = None,
                    create_time: int = None,
                    update_time: int = None,
                    ) -> WorkflowMeta:
    return WorkflowMeta(name=name, project_id=project_id, properties=properties,
                        create_time=create_time, update_time=update_time)


import logging
import queue

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO, format='%(message)s')

QUEUE_TIMEOUT = 10 ** -2

class LambdaWorker(object):
    '''Simple worker that waits for jobs, runs them, and returns results.'''

    def __init__(self, job_queue, result_dict, worker_id):
        self.functions = {}
        self._job_queue = job_queue
        self._result_dict = result_dict
        self._id = worker_id
        self.await_jobs()

    def await_jobs(self):
        '''Waits for new jobs to come in via the job queue, runs the job, and
        puts the result into the results dict.'''

        logger.info('[WORKER {}] Ready!'.format(self._id))

        while True:
            try:
                job = self._job_queue.get(timeout=QUEUE_TIMEOUT)
            except queue.Empty:
                logger.debug('[WORKER {}] found empty queue'.format(self._id))
                continue

            logger.debug('[WORKER {}] Running job {}'
                         .format(self._id, job.job_id))
            result = job.run()

            logger.debug('[WORKER {}] Returning result {}'
                         .format(self._id, job.job_id))
            self._result_dict[job.job_id] = result


"""IMDB class file"""
import requests

class IMDB:
    """Class to retrieve movie info using apache64"""

    def __init__(self, title, field=""):
        """Instance variables"""
        self.title = title
        self._field = field

    @property
    def field(self):
        return self._field

    @field.setter
    def field(self, input_field):
        """Returns IMDB field for the instantiated movie"""
        try:
            title = self.title.strip().replace(' ', '+')
            imdb = 'http://theapache64.xyz:8080/movie_db/search?keyword={0}'.format(title)
            json = requests.Session().get(imdb).json()
            value = json['data'][input_field]
        except KeyError:
            value = "N/A"
        self._field = value


from chroniclepy.chroniclepy import summarising, preprocessing, subsetting
from argparse import ArgumentParser
import pandas as pd
import unittest
import shutil
import yaml
import os

class PreprocessingOutputTest(unittest.TestCase):

    def test_preprocessing(self):
        print("Test preprocessing simple")
        preprocessing.preprocess_folder(
            infolder='resources/rawdata',
            outfolder='resources/preprocessed'
        )
        self.assertTrue(True)

    def test_output(self):
        print("Test preprocessing output")

        directory = '.'

        # preprocessed
        files = {
            "preprocessing": {
                "expected": os.path.join(
                    directory,
                    'resources/constants/constants_preprocessed.yaml'
                ),
                "encountered": os.path.join(
                    directory,
                    'resources/preprocessed/ChronicleData_preprocessed-TestParticipant.csv'
                ),
                "folder": os.path.join(directory, "resources/preprocessed")
            },
            "summarising": {
                "expected": os.path.join(
                    directory,
                    'resources/constants/constants_summary.yaml'
                ),
                "encountered": os.path.join(
                    directory,
                    'resources/output/summary_daily.csv'
                ),
                "folder": os.path.join(directory, "resources/output")
            }
        }

        for keyword in files.keys():

            print("Testing pipeline for %s"%keyword)

            with open(files[keyword]['expected'], 'r') as outfile:
                expected_output = yaml.safe_load(outfile)

            encountered_output = pd.read_csv(files[keyword]['encountered']).to_dict()
            if not encountered_output == expected_output:
                print("    There are files with different values !!")
                for key, value in expected_output.items():
                    if value != encountered_output[key]:
                        print("        Difference for %s: %s vs. %s"%(key, str(value[0]), str(encountered_output[key][0])))

                raise ValueError("    This isn't right !")
            else:
                print("    Test successful...")
                shutil.rmtree(files[keyword]['folder'])

if __name__ == '__main__':
    unittest.main()


#python first
#django second
#your apps
#locals directory

from django.conf import settings
from django.contrib import messages
from django.core.mail import send_mail
from django.shortcuts import render, render_to_response, RequestContext, get_object_or_404, HttpResponseRedirect
from .models import Product

#MessagesCommands

#messages.debug(request, '' % count)
#messages.info(request, '')
#messages.success(request, '')
#messages.warning(request, '')
#messages.error(request, '')

# Create your views here.
from cart.forms import ProductQtyForm
from .forms import Product

def all_products(request):
	products = Product.objects.all()

	return render_to_response("products/all.html",
								locals(),
								context_instance=RequestContext(request))

def single_product(request, slug):
	add_product = ProductQtyForm()
	product = get_object_or_404(Product, slug=slug)

	return render_to_response("products/single.html",
								locals(),
								context_instance=RequestContext(request))

def home(request):
	products = Product.objects.all()

	return render_to_response("home.html",
								locals(),
								context_instance=RequestContext(request))

def thankyou(request):

	form = ProductFrom(request.POST or None)

	if form.is_valid():
		save_it = form.save(commit=False)
		save_it.save()

		#send_mail(subject, message, from_email, to_list, fail_silently=True)
		subject = 'You Have Succesfully Created a Product'

		return HttpResponseRedirect('/thank-you/')

	return render_to_response("thankyou.html",
								locals(),
								context_instance=RequestContext(request))

def aboutus(request):

	return render_to_response("aboutus.html",
								locals(),
								context_instance=RequestContext(request))

def contactus(request):

	return render_to_response("contactus.html",
								locals(),
								context_instance=RequestContext(request))

def search(request):
	try:
		q = request.GET.get('q', '')
	except:
		q = False

	if q:
		k = q.split()
		if len(k) >= 2:
			products = []
			for i in k:
				all_products = Product.objects.filter(name__icontains=i).distinct()
				for product in all_products:
					if product not in products:
						products.append(product)
		else:
			products = Product.objects.filter(name__icontains=q)

	return render_to_response("products/search.html",
								locals(),
								context_instance=RequestContext(request))


#!/usr/bin/env python

import rospy
import sys
import tf
# from control_msgs.msg import GripperCommandAction
from giskardpy.python_interface import GiskardWrapper
from giskardpy.utils_constraints import Utils
from geometry_msgs.msg import PoseStamped, Point, Quaternion
from iai_naive_kinematics_sim.srv import SetJointState, SetJointStateRequest, SetJointStateResponse  # comment this line
from sensor_msgs.msg import JointState
import numpy as np
from tf.transformations import quaternion_about_axis
import tf2_ros
import tf2_geometry_msgs
from giskardpy.qp_problem_builder import SoftConstraint
from collections import OrderedDict
from geometry_msgs.msg import Vector3Stamped, Vector3
from rospy_message_converter.message_converter import convert_dictionary_to_ros_message
from giskardpy import constraints as cnst
from giskardpy import god_map as gm
from giskardpy import tfwrapper as tf_wrapper
from giskardpy import symbolic_wrapper as w
from urdf_parser_py.urdf import URDF
import math
import yaml
from giskardpy import utils_constraints as uc
from giskardpy.tfwrapper import lookup_pose, pose_to_kdl, np_to_kdl, kdl_to_pose
import actionlib
from control_msgs.msg import GripperCommandAction, GripperCommandGoal
from pr2_controllers_msgs.msg import Pr2GripperCommandAction, Pr2GripperCommandActionGoal, Pr2GripperCommandGoal

class MotionTaskWithConstraint:
    """
    the class helps to select the necessary constraints, to select the gripper and to check the orientation of the base.
    """
    # list of Gripper
    _list_grippers = []
    # Torso of robot
    _torso = ""
    # Reference frame to basefootprint
    _origin = ""
    # goal frame
    _goal = ""

    def __init__(self):
        print("MotionTaskWithConstraint is initialized !")
        self._giskard = GiskardWrapper()

        rospy.logout("- Set pose kitchen")
        kitchen_pose = tf_wrapper.lookup_pose("map", "iai_kitchen/world")
        kitchen_pose.header.frame_id = "map"
        # setup kitchen
        rospy.logout("- Get urdf")
        self.urdf = rospy.get_param("kitchen_description")
        self.parsed_urdf = URDF.from_xml_string(self.urdf)
        self.config_file = None

        rospy.logout("- clear urdf and add kitchen urdf")
        self._giskard.clear_world()
        self._giskard.add_urdf(name="kitchen", urdf=self.urdf, pose=kitchen_pose, js_topic="kitchen/cram_joint_states")

    def set_gripper_for_real_roboter(self):
        # setup gripper as actionlib
        self.rgripper = actionlib.SimpleActionClient('r_gripper/gripper_command', GripperCommandAction)
        self.lgripper = actionlib.SimpleActionClient('l_gripper/gripper_command', GripperCommandAction)
        self.rgripper.wait_for_server()
        self.lgripper.wait_for_server()

    def r_action_gripper(self, value=0):
        goal = Pr2GripperCommandGoal() #GripperCommandGoal()
        goal.command.position = value
        goal.command.max_effort = 40
        self.rgripper.send_goal_and_wait(goal)

    def l_action_gripper(self, value=0):
        goal = Pr2GripperCommandGoal() #GripperCommandGoal()
        goal.command.position = value
        goal.command.max_effort = 40
        self.lgripper.send_goal_and_wait(goal)

    def execute_open_translational_motion_on_real_robot(self, goal, body, action, joint, joint_value):
        self.r_action_gripper(0.08)
        self.l_action_gripper(0.08)
        self._giskard.set_json_goal("MoveToPoseConstraint", goal_name=goal, body_name=body)
        self._giskard.allow_all_collisions()
        self._giskard.plan_and_execute()
        self.r_action_gripper(0.012)
        self.l_action_gripper(0.012)
        self._giskard.set_json_goal("OpenCloseDrawerConstraint", goal_name=goal, body_name=body, action=action)
        self._giskard.allow_all_collisions()
        # self.giskard.avoid_all_collisions()
        self._giskard.plan_and_execute()
        self._giskard.set_object_joint_state(object_name='kitchen',
                                             joint_states={joint: joint_value})

    def execute_close_translational_motion_on_real_robot(self, goal, body, action, joint, joint_value):
        self.r_action_gripper(0.08)
        self.l_action_gripper(0.08)
        self._giskard.set_json_goal("MoveToPoseConstraint", goal_name=goal, body_name=body)
        self._giskard.allow_all_collisions()
        self._giskard.plan_and_execute()
        self._giskard.set_json_goal("OpenCloseDrawerConstraint", goal_name=goal, body_name=body, action=action)
        self._giskard.allow_all_collisions()
        # self.giskard.avoid_all_collisions()
        self._giskard.plan_and_execute()
        self._giskard.set_object_joint_state(object_name='kitchen',
                                             joint_states={joint: joint_value})

    def execute_open_circular_motion_on_real_robot(self, goal, body, action, joint, joint_value):
        self.r_action_gripper(0.08)
        self.l_action_gripper(0.08)
        self._giskard.set_json_goal("MoveToPoseConstraint", goal_name=goal, body_name=body)
        self._giskard.allow_all_collisions()
        self._giskard.plan_and_execute()
        self.r_action_gripper(0.012)
        self.l_action_gripper(0.012)
        self._giskard.set_json_goal("OpenCloseDoorConstraint", goal_name=goal, body_name=body, action=action)
        self._giskard.allow_all_collisions()
        # self.giskard.avoid_all_collisions()
        self._giskard.plan_and_execute()
        self._giskard.set_object_joint_state(object_name='kitchen',
                                             joint_states={joint: joint_value})

    def execute_close_circular_motion_on_real_robot(self, goal, body, action, joint, joint_value):
        self.r_action_gripper(0.08)
        self.l_action_gripper(0.08)
        self._giskard.set_json_goal("MoveToPoseConstraint", goal_name=goal, body_name=body)
        self._giskard.allow_all_collisions()
        self._giskard.plan_and_execute()
        self._giskard.set_json_goal("OpenCloseDoorConstraint", goal_name=goal, body_name=body, action=action)
        self._giskard.allow_all_collisions()
        # self.giskard.avoid_all_collisions()
        self._giskard.plan_and_execute()
        self._giskard.set_object_joint_state(object_name='kitchen',
                                             joint_states={joint: joint_value})

    def execute_open_rotational_motion_on_real_robot(self, goal, body, action, joint, joint_value):
        self.r_action_gripper(0.08)
        self.l_action_gripper(0.08)
        self._giskard.set_json_goal("MoveToPoseConstraint", goal_name=goal, body_name=body)
        self._giskard.allow_all_collisions()
        self._giskard.plan_and_execute()
        self.r_action_gripper(0.012)
        self.l_action_gripper(0.012)
        self._giskard.set_json_goal("TurnRotaryKnobConstraint", goal_name=goal, body_name=body, action=action)
        self._giskard.allow_all_collisions()
        # self.giskard.avoid_all_collisions()
        self._giskard.plan_and_execute()
        self._giskard.set_object_joint_state(object_name='kitchen',
                                             joint_states={joint: joint_value})

    def execute_close_rotational_motion_on_real_robot(self, goal, body, action, joint, joint_value):
        self.r_action_gripper(0.08)
        self.l_action_gripper(0.08)
        self._giskard.set_json_goal("MoveToPoseConstraint", goal_name=goal, body_name=body)
        self._giskard.allow_all_collisions()
        self._giskard.plan_and_execute()
        self._giskard.set_json_goal("TurnRotaryKnobConstraint", goal_name=goal, body_name=body, action=action)
        self.giskard.allow_all_collisions()
        # self.giskard.avoid_all_collisions()
        self.giskard.plan_and_execute()
        self._giskard.set_object_joint_state(object_name='kitchen',
                                            joint_states={joint: joint_value})

    def setup_orientation_basis(self, basis_frame="odom_combined", torso="torso_lift_link",
                                list_grippers=["r_gripper_tool_frame", "l_gripper_tool_frame"]):
        """
        This method set the parameter for the orientations verification of the basis
        :param basis_frame: origin frame to the basis
        :type: str
        :param torso: torso of robot
        :type: str
        :param list_grippers: List of gripper
        :type: array of str
        :return:
        """
        self._list_grippers = list_grippers
        self._torso = torso
        self._origin = basis_frame

    def set_goal(self, goal=""):
        """
        set the goal in relation to which the base is to be oriented
        :param goal: goal
        :type: str
        :return:
        """
        self._goal = goal

    def update_basis_orientation(self):
        """
        This function update the orientation of the basis and get the next Gripper in short distance from goal
        :return:
        """
        x, y, r = self.get_current_base_position()
        rospy.logout("The basis is at position :")
        rospy.logout(x)
        rospy.logout(y)
        rospy.logout(r)
        # get pose torso
        pose_torso = tf_wrapper.lookup_pose(self._origin, self._torso)
        # get pose goal
        pose_goal = tf_wrapper.lookup_pose(self._origin, self._goal)
        # get list of pose of gripper
        list_pose_gripper = [tf_wrapper.lookup_pose(self._origin, p) for p in self._list_grippers]
        # get angle to G
        angle = [float(w.get_angle_casadi(self.get_x_y(pose_torso), self.get_x_y(pose_goal), self.get_x_y(p)))
                 for p in list_pose_gripper]
        # max angle G to goal
        max_angle = np.amax(angle)
        rospy.logout("Max angle is :")
        rospy.logout(max_angle)
        # update old and new orientation
        max_angle = r + max_angle
        rospy.logout("Max angle + r is :")
        rospy.logout(max_angle)
        # get selected Gripper
        gripper = np.where(np.array(angle) == np.amax(angle))
        rospy.logout("Farther away Gripper:")
        rospy.logout(gripper[0][0])
        # convert angle in axis
        q = w.quaternion_from_axis_angle([0, 0, 1], max_angle)
        # update basis orientation
        #self.set_cart_goal_for_basis(x, y, 0, q)

    #@staticmethod
    def get_x_y(self, pose):
        return [pose.pose.position.x, pose.pose.position.y, 0]

    def get_current_base_position(self):
        """
        the method check the current pose of base_footprint to odom and get it
        :return: x, y, rotation
        """
        basis = tf_wrapper.lookup_pose(self._origin, "base_footprint")
        t, r = [basis.pose.position.x, basis.pose.position.y, basis.pose.position.z], \
               [basis.pose.orientation.x, basis.pose.orientation.y, basis.pose.orientation.z,
                basis.pose.orientation.w]  # self.get_msg_translation_and_rotation(self._basis, "map")
        return self.base_position(t, r)

    def base_position(self, translation, rotation):
        """
        the method take the translation and rotation from tf_ros and calculate the rotation in euler
        :param translation: vector
        :param rotation: quaternion (from tf_ros)
        :return: vector[x, y, rotation], current base position
        """
        euler = tf.transformations.euler_from_quaternion(rotation)
        return translation[0], translation[1], euler[2]

    def set_cart_goal_for_basis(self, x, y, z, q):
        """
        this function send goal to the basis of robot
        :param x: position on x axis
        :type: float
        :param y: position on y axis
        :type: float
        :param z: position on z axis
        :type: float
        :param q: quaternion orientation
        :type: array float
        :return:
        """
        # take base_footprint
        poseStamp = lookup_pose(self._origin, "base_footprint")
        poseStamp.header.frame_id = self._origin
        # set goal pose
        poseStamp.pose.position.x = x
        poseStamp.pose.position.y = y
        poseStamp.pose.position.z = z
        poseStamp.pose.orientation.x = q[0]
        poseStamp.pose.orientation.y = q[1]
        poseStamp.pose.orientation.z = q[2]
        poseStamp.pose.orientation.w = q[3]
        self._giskard.set_cart_goal(self._origin, "base_footprint", poseStamp)
        self._giskard.avoid_all_collisions()
        self._giskard.plan_and_execute()

    def reset_kitchen(self, list_joints):
        for j in list_joints:
            self._giskard.set_object_joint_state(object_name='kitchen',
                                                 joint_states={j: 0.0})

if __name__ == '__main__':
    rospy.init_node('do_test_constraint')
    list_joint = [
        "sink_area_trash_drawer_main_joint",
        "sink_area_left_upper_drawer_main_joint",
        "sink_area_left_middle_drawer_main_joint",
        "sink_area_left_bottom_drawer_main_joint",
        "sink_area_dish_washer_main_joint",
        "sink_area_dish_washer_door_joint",
        "oven_area_oven_door_joint",
        "oven_area_oven_knob_stove_1_joint",
        "oven_area_oven_knob_stove_2_joint",
        "oven_area_oven_knob_stove_3_joint",
        "oven_area_oven_knob_stove_4_joint",
        "oven_area_oven_knob_oven_joint",
        "oven_area_area_middle_upper_drawer_main_joint",
        "oven_area_area_middle_lower_drawer_main_joint",
        "oven_area_area_left_drawer_main_joint",
        "oven_area_area_right_drawer_main_joint",
        "kitchen_island_left_upper_drawer_main_joint",
        "kitchen_island_left_lower_drawer_main_joint",
        "kitchen_island_middle_upper_drawer_main_joint",
        "kitchen_island_middle_lower_drawer_main_joint",
        "kitchen_island_right_upper_drawer_main_joint",
        "kitchen_island_right_lower_drawer_main_joint",
        "fridge_area_lower_drawer_main_joint",
        "iai_fridge_door_joint"
    ]
    rospy.logout("START SOME MOVE as Test")
    mc = MotionTaskWithConstraint()
    mc.set_gripper_for_simulator()
    # fridge door
    #mc.execute_open_circular_motion_on_real_robot('iai_kitchen/iai_fridge_door_handle', 'r_gripper_tool_frame', 0.5,
                                                 #'iai_fridge_door_joint', 0.758)
    #mc.execute_close_circular_motion_on_real_robot('iai_kitchen/iai_fridge_door_handle', 'r_gripper_tool_frame', -0.5,
                                                  #'iai_fridge_door_joint', 0.0)

    #mc.reset_kitchen(list_joint)

    # kitchen_island_left_upper_drawer
    #mc.execute_open_translational_motion_on_real_robot("iai_kitchen/kitchen_island_left_upper_drawer_handle",
                                                      #"r_gripper_tool_frame", 1,
                                                      #"kitchen_island_left_upper_drawer_main_joint", 0.48)
    #mc.execute_close_translational_motion_on_real_robot("iai_kitchen/kitchen_island_left_upper_drawer_handle",
                                                       #"r_gripper_tool_frame", -1,
                                                       #"kitchen_island_left_upper_drawer_main_joint", 0.0)
    #mc.reset_kitchen(list_joint)

    # iai_kitchen/oven_area_oven_door_handle
    #mc.execute_open_circular_motion_on_real_robot('iai_kitchen/oven_area_oven_door_handle', 'l_gripper_tool_frame', -1,
                                                 #'oven_area_oven_door_joint', 1.57)
    #mc.execute_close_circular_motion_on_real_robot('iai_kitchen/oven_area_oven_door_handle', 'l_gripper_tool_frame', 1,
                                                  #'oven_area_oven_door_joint', 0.0)
    #mc.reset_kitchen(list_joint)

    # iai_kitchen/sink_area_dish_washer_door_handle
    mc.execute_open_circular_motion_on_real_robot('iai_kitchen/sink_area_dish_washer_door_handle',
                                                 'r_gripper_tool_frame',
                                                 -0.5, 'sink_area_dish_washer_door_joint', 0.758)
    mc.execute_close_circular_motion_on_real_robot('iai_kitchen/sink_area_dish_washer_door_handle',
                                                  'r_gripper_tool_frame',
                                                  0.5, 'sink_area_dish_washer_door_joint', 0)
    #mc.reset_kitchen(list_joint)

    # iai_kitchen/sink_area_left_middle_drawer_handle
    #mc.execute_open_translational_motion_on_real_robot('iai_kitchen/sink_area_left_middle_drawer_handle',
                                                      #'r_gripper_tool_frame', -1,
                                                      #'sink_area_left_middle_drawer_main_joint', 0.48)
    #mc.execute_close_translational_motion_on_real_robot('iai_kitchen/sink_area_left_middle_drawer_handle',
                                                       #'r_gripper_tool_frame', 1,
                                                       #'sink_area_left_middle_drawer_main_joint', 0.0)
    #mc.reset_kitchen(list_joint)

    # iai_kitchen/oven_area_oven_knob_stove_4
    #mc.execute_rotational_motion_on_real_robot('iai_kitchen/oven_area_oven_knob_stove_4', 'l_gripper_tool_frame', -1,
                                              #"oven_area_oven_knob_stove_4_joint", 1.57)

    # set_cart_goal_test()
    rospy.spin()