__author__ = 'Hannnn'

# !/usr/bin/python
# -*- coding: cp1251 -*-

from os.path import join
import json
from io import open
import turtle

class JSONFile:
    __file = None
    __filepath = None

    def __init__(self, file_path, file_name):
        self.__filepath = join(file_path, file_name)

    def create_file(self):
        self.__file = open(self.__filepath, 'w+', encoding='utf-8')

    def open_file(self):
        self.__file = open(self.__filepath, 'r+', encoding='utf-8')

    def read_file(self):
        return json.loads(self.__file.read())

    def save_to_file(self, data_save):
        self.__file.seek(0)
        self.__file.write(unicode(json.dumps(data_save)))
        self.__file.truncate()

    def added_to_file(self, data):
        values = self.read_file()
        values.append(json.loads(data))
        self.save_to_file(values)

    def is_create(self):
        return self.__file

    def __del__(self):
        if self.__file:
            self.__file.close()
            del self.__file

class Figure:
    __name = None
    __color = None
    __coordinates = []

    def __init__(self, name, color, coordinates):
        self.set_info(name, color, coordinates)

    def set_info(self, name, color, coordinates):
        self.__name = name
        self.__color = color
        self.__coordinates = coordinates

    def json(self):
        json_values = {'name': self.__name, 'color': self.__color, 'coordinate': self.__coordinates}

        return json.dumps(json_values)

    def show(self):
        print('Name: ', self.__name, '\n', 'color', self.__color, '\n', 'Coordinate: ',
              self.__coordinates, '\n')

class Figures(JSONFile):
    __array = []  # Figures list

    def count(self):
        return len(self.__array)

    def add(self, figure):
        self.__array.append(figure.json())

    def save(self):
        self.save_to_file(self.__array)

    def read(self):
        self.__array = self.read_file()

    def get(self, fid):
        return self.__array[fid]

    def change(self, fid, data):
        self.__array[fid] = data

    def all(self):
        return self.__array

    def show_all(self):
        i = 0
        for item in self.__array:
            print(i, item)
            i += 1

class Draw(object):
    window = None
    panel = None
    panel_size = 0
    color = None

    def __init__(self, panel_size, bcolor):
        self.panel_size = panel_size
        self.color = bcolor

    def draw_json(self, figures):
        self.window = turtle.Screen()
        self.window.setup(self.panel_size, self.panel_size)
        self.panel = turtle.Turtle()
        self.panel.color(self.color)
        self.window.delay()

        self.panel.showturtle()
        self.panel.shape('turtle')
        self.panel.clear()   # clear panel before start
        for first in figures:
            first = json.loads(first)
            self.panel.color(first['color'])
            self.panel.fillcolor(first['color'])
            self.panel.begin_fill()
            for second in first['coordinate']:
                self.panel.setpos(second[0], second[1])
                self.panel.dot(5)

            self.panel.home()
            self.panel.end_fill()

        self.panel.shape('blank')

        self.window.exitonclick()

class Console(object):
    option = 0
    obj = None

    figures = None

    @staticmethod
    def input_coordinates():
        coordinates = []

        print('Write figure coordinates: ')

        flag_coordinate = True

        while flag_coordinate:
            x = int(raw_input('Write x: '))
            y = int(raw_input('Write y: '))

            coordinates.append([x, y])

            if raw_input('Continue? [Y|N]') == 'n':
                flag_coordinate = False

        return coordinates

    def add_figure(self):
        name = raw_input('Write figure name: ')
        color = raw_input('Write figure color: ')

        coordinates = self.input_coordinates()

        return Figure(name, color, coordinates)

    def change_figure(self):
        fid = int(raw_input('Write figure number: '))

        new_value = json.loads(self.figures.get(fid))

        name = new_value['name']
        color = new_value['color']
        coordinate = new_value['coordinate']

        if raw_input('Change name?: [Y|N] ') == 'y':
            name = raw_input('Write name: ')

        if raw_input('Change color?: [Y|N] ') == 'y':
            color = raw_input('Write color: ')

        if raw_input('Change coordinates?: [Y|N] ') == 'y':
            coordinate = self.input_coordinates()

        self.figures.change(fid, Figure(name, color, coordinate).json())
        self.figures.save()

    def menu(self):

        # path = raw_input('Write file path: ')
        # name = raw_input('Write file name: ')

        path = '/home/user/'
        name = 'testet.txt'

        self.figures = Figures(path, name)  # Create file path

        flag = True

        while flag:
            print('::::Choose menu item::::')
            print('1- Create file')
            print('2- Open file [Edit]')
            print('3- Save file: ')
            print('4- Add figure')
            print('5- Show all figures list')
            print('6- Change figure #')
            print('7- Draw')
            print('0- EXIT')
            print('::::::::::::::::::::::::')

            try:
                self.option = int(raw_input('Write item number: '))  # Get item number

                # File create
                if self.option == 1:
                    print('Command:: Create')
                    self.figures.create_file()
                # Open file
                elif self.option == 2:
                    print('Command:: Open')
                    self.figures.open_file()
                    self.figures.read()
                # Save file
                elif self.option == 3:
                    print('Command:: Save')

                    if not self.figures.is_create():  # file created?
                        print('File error: use item 1 or 2')
                    elif not self.figures.count():  # > 0
                        print('File save error: use item 4')
                    else:
                        self.figures.save()
                # New figure
                elif self.option == 4:
                    self.figures.add(self.add_figure())
                # Show all figures
                elif self.option == 5:
                    self.figures.show_all()
                # Change parameters
                elif self.option == 6:
                    self.figures.show_all()
                    print('\n')
                    self.change_figure()
                # Draw
                elif self.option == 7:
                    draw = Draw(900, 'lightgreen')
                    draw.draw_json(self.figures.all())
                    # self.figures.draw(obj)0
                elif self.option == 0:
                    print('Command:: Exit')
                    flag = False
            except:
                print('Command:: Exit')
                flag = False

            print('::::::::::::::::::::::::\n')

# Run console mod
console = Console()

# Start
console.menu()


from __future__ import print_function
import nltk, re, pprint
from nltk import word_tokenize
from nltk.corpus import stopwords
#nltk.corpus.stopwords.words('danish')
from nltk.stem import *
from nltk.stem.porter import *
from collections import Counter
import os, sys

#path to folder containing the raw scrapes from either kk.dk or minby.dk - each file represents a district
path = 'C:\\Users\\simo7\\Documents\\GitHub\\CPH_Neighborhood_NLP_Project\\2nd_search\\Resources_Danish\\kk_scrapes'

dirs = os.listdir(path)

def tokenizer(filename):
    with open("C:\\Users\\simo7\\Documents\\GitHub\\CPH_Neighborhood_NLP_Project\\2nd_search\\Resources_Danish\\kk_scrapes\\" + filename) as file:
        raw_content = []
        while True:

            l=file.readline()

            if len(l) == 0:
                break
            raw_line = l.strip().lower()
            raw_content.append(raw_line)

        #a list of all cleaned and stemmed words in the scraped file
        all_data = []

        for article in raw_content:
            #splits all words into items in a list
            tokens = word_tokenize(article)
            #removes all stopwords from the nltk library
            stopWords = set(stopwords.words('danish'))
            tokens_no_stopwords = [w for w in tokens if w not in stopWords]

            #removes items containing commas ect.
            nonPunct = re.compile('.*[A-Za-z0-9].*')
            filtered = [w for w in tokens_no_stopwords if nonPunct.match(w)]

            #At this point some tokens still contained numbers, as well as strings suchs as "Lyngby.Svømmehal"
            #super_clean removes this
            super_clean = []
            for word in filtered:
                dummy = 0
                for char in word:
                    if char == ".":
                        #splits words like "amager.Hej" by punctuation
                        splitted = word.split(".")
                        #removes digits and small words for either the first or second split
                        if len(splitted[0]) > 2 and splitted[0].isdigit() == False:
                            super_clean.append(splitted[0])
                        if len(splitted[1]) > 2 and splitted[1].isdigit() == False:
                            super_clean.append(splitted[1])
                        dummy = 1
                        break
                    if char.isdigit() == True:
                        dummy = 1
                        break
                if dummy == 0:
                    if len(word) > 1:
                        super_clean.append(word)

            #counts = Counter(filtered)
            text = nltk.Text(tokens)
            #collocations = text.collocations()
            #return counts
            all_data.append(super_clean)

        return all_data

#This script stemmes all words - stopwords have already been removed
def stemmer(tokens):
    stemmer_settings = SnowballStemmer("danish", ignore_stopwords=True)
    stemmed_tokens = [stemmer_settings.stem(plural) for plural in tokens]
    return stemmed_tokens

#runs the cleaning script for all files/ areas, and writes the output to new seperate files
for scrape_file in dirs:
    new_name = scrape_file.split(".")[0] + "_stemmed(no_stopwords).txt"
    new_file = open(new_name,"w+")
    scrape_tokens = tokenizer(scrape_file)

    for list in scrape_tokens:
        stemmed_scrape = stemmer(list)
        for idx,word in enumerate(stemmed_scrape):
            if idx != 0:
                new_file.write(",")
            new_file.write(word)
        new_file.write("\n")


from rest_framework.relations import PrimaryKeyRelatedField
from sharebookApp.models.author import Author
from sharebookApp.models.book import Book
from sharebookApp.models.grade import Grade
from sharebookApp.models.editorial import Editorial
from rest_framework import serializers

from sharebookApp.models.user import User

class BookSerializer(serializers.ModelSerializer):
    editorial = PrimaryKeyRelatedField(queryset=Editorial.objects.all())
    grade = PrimaryKeyRelatedField(queryset=Grade.objects.all())
    author = PrimaryKeyRelatedField(queryset=Author.objects.all(),many=True)
    user = PrimaryKeyRelatedField(queryset=User.objects.all())

    class Meta:
        model = Book
        fields = ['id_book', 'isbn', 'title', 'language',
                  'price', 'state', 'editorial', 'grade', 'author', 'user']

    def create(self, validated_data):
        author = validated_data.pop("author")
        instance = Book.objects.create(**validated_data)
        instance.author.set(author)
        return instance

    def to_representation(self, obj):
        book = Book.objects.get(id_book=obj.id_book)
        editorial = Editorial.objects.get(id_editorial=obj.editorial_id)
        grade = Grade.objects.get(id_grade=obj.grade_id)
        user = User.objects.get(id=obj.user_id)

        authors = []
        for author in obj.author.all():
            authors.append(author.name_author)

        return {
            'id_book': book.id_book,
            'isbn': book.isbn,
            'title': book.title,
            'language': book.language,
            'price': book.price,
            'state': book.state,
            'editorial':  editorial.name_editorial,
            'grade': grade.name_grade,
            'author': authors,
            'user': user.name,
        }


#!/usr/bin/python3
"""new view for User objects that handles
all default RestFul API actions"""

from flask import Flask, jsonify, abort, request
from api.v1.views import app_views
from models import storage
from models.user import User

@app_views.route('/users', methods=['GET'], strict_slashes=False)
def get_allUsers():
    """Retrieve a list of Users objects"""
    obj_dic = storage.all("User")
    return jsonify([users.to_dict() for users in obj_dic.values()])

@app_views.route('/users/<user_id>', methods=['GET'],
                 strict_slashes=False)
def get_user(user_id):
    """Retrieve a specific user object by id"""
    user = storage.get(User, user_id)
    if user:
        return jsonify(user.to_dict())
    else:
        abort(404)

@app_views.route('/users/<user_id>', methods=['DELETE'],
                 strict_slashes=False)
def del_user(user_id):
    """Delete a specific user object by id"""
    user = storage.get(User, user_id)
    if user:
        storage.delete(user)
        storage.save()
        return jsonify({}), 200
    else:
        abort(404)

@app_views.route('/users', methods=['POST'], strict_slashes=False)
def create_user():
    """Create a user object"""
    if request.get_json() is None:
        return "Not a JSON", 400
    elif 'email' not in request.get_json():
        return "Missing email", 400
    elif 'password' not in request.get_json():
        return "Missing password", 400
    else:
        user = User(**request.get_json())
        user.save()
        return jsonify(user.to_dict()), 201

@app_views.route('/users/<user_id>', methods=['PUT'],
                 strict_slashes=False)
def uptade_user(user_id):
    """Update a user store into storage"""
    if request.get_json() is None:
        return "Not a JSON", 400
    user = storage.get(User, user_id)
    if user:
        ignore_keys = ['id', 'email', 'created_at', 'updated_at']
        for key, value in request.get_json().items():
            if key not in ignore_keys:
                setattr(user, key, value)
        user.save()
        return jsonify(user.to_dict()), 200
    else:
        abort(404)


# coding: utf-8

# # Preprocessing

# In[279]:

from string import punctuation
import numpy as np
def clean_text(data_list):
    lines = list()
    for i in range(2000):
        line = str(data_list[i]).split()
        table = str.maketrans('', '', punctuation)
        line = [w.translate(table) for w in line]
        line  = [word for word in line if word.isalpha() and len(word) > 1]
        lines.append(' '.join(line))
    return lines

# In[280]:

import sklearn.datasets

# load text files and process
docs_to_train = sklearn.datasets.load_files("./txt_sentoken",shuffle = False)
data = clean_text(docs_to_train.data)

# In[281]:

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import train_test_split

# select words with min occurance >=5
# and does not appear in more than 80% of the files
# filter stop words
vectorizer = CountVectorizer(min_df = 5,max_df= 0.8,stop_words='english')
X = vectorizer.fit_transform(data)
y = docs_to_train.target

# train test split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42,shuffle = True)

# # Uncertainty Sampling

# In[272]:

from sklearn.metrics import accuracy_score
from sklearn.svm import LinearSVC
K = 10
T = X_train.todense()[0:100]
T_y = y_train[0:100]
T_prime = np.delete(X_train.todense(),list(range(100)),axis = 0)
T_prime_y = y_train[100:1600]
T_len = len(T_prime_y)
test_errors = []
train_size = []

while T_len >= 0:
    clf = LinearSVC()
    clf.fit(T,T_y)
    if (len(T_prime_y) > K):
        # selecting examples closest to the hyperplane
        idx = np.argpartition(np.abs(clf.decision_function(T_prime)), K)[0:K]
    else:
        idx = list(range(K))

    y_pred = clf.predict(X_test.todense())
    error = 1- accuracy_score(y_test, y_pred)
    test_errors.append(error)
    train_size.append(len(T))

    if len(T_prime_y) > 0:
        for i in idx:
            T = np.vstack([T,T_prime[i]])
            T_y = np.append(T_y, T_prime_y[i])
        T_prime = np.delete(T_prime,[i for i in idx],axis = 0)
        T_prime_y = np.delete(T_prime_y,[i for i in idx])
    T_len -= K

# In[271]:

import matplotlib.pyplot as plt
fig, ax = plt.subplots()
ax.plot(train_size,test_errors)

ax.set(xlabel='training set size', ylabel='test error',
        title='Uncertainty Sampling')

# helper function to display point with lowest y value
def annot_min(x,y, ax=None):
    xmin = x[np.argmin(y)]
    ymin = min(y)
    text= "x={:.3f}, y={:.3f}".format(xmin, ymin)
    if not ax:
        ax=plt.gca()
        bbox_props = dict(boxstyle="square,pad=0.3", fc="w", ec="k", lw=0.72)
        arrowprops=dict(arrowstyle="->",connectionstyle="angle,angleA=0,angleB=60")
        kw = dict(xycoords='data',textcoords="axes fraction",
                  arrowprops=arrowprops, bbox=bbox_props, ha="right", va="top")
        ax.annotate(text, xy=(xmin, ymin), xytext=(0.94,0.96), **kw)

annot_min(train_size,test_errors)

plt.show()

# # Query by Committee

# In[ ]:

from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import MultinomialNB
from sklearn import tree

K= 10
T = X_train.todense()[0:100]
T_y = y_train[0:100]
T_prime = np.delete(X_train.todense(),list(range(100)),axis = 0)
T_prime_y = y_train[100:1600]
T_len = len(T_prime_y)
test_errors = []
train_size = []
while T_len >= 0:
    knn_clf = KNeighborsClassifier(n_neighbors = 1)
    knn_clf.fit(T,T_y)
    nb_clf = MultinomialNB()
    nb_clf.fit(T,T_y)
    tree_clf = tree.DecisionTreeClassifier(max_depth = 30)
    tree_clf.fit(T,T_y)

    knn_pred = knn_clf.predict(X_test.todense())
    nb_pred = nb_clf.predict(X_test.todense())
    tree_pred = tree_clf.predict(X_test.todense())

    train_size.append(len(T))
    print(len(T))

    majority_pred = []
    idx = []

    for i in range(len(y_test)):
        votes = np.array([knn_pred[i],nb_pred[i],tree_pred[i]])
        majority_pred.append(np.argmax(np.bincount(votes)))

        #vote entropy
        if len(idx) < K:
            if np.count_nonzero(votes == 0) == 2 or np.count_nonzero(votes == 1) == 2:
                idx.append(i)

    error = 1- accuracy_score(y_test, majority_pred)
    print(error)
    test_errors.append(error)
    if len(T_prime_y) > K:
        for i in idx:
            T = np.vstack([T,T_prime[i]])
            T_y = np.append(T_y, T_prime_y[i])
        T_prime = np.delete(T_prime,[i for i in idx],axis = 0)
        T_prime_y = np.delete(T_prime_y,[i for i in idx])

        while len(idx) < K:
            T = np.vstack([T,T_prime[0]])
            T_y = np.append(T_y, T_prime_y[0])
            T_prime = np.delete(T_prime,[0],axis = 0)
            T_prime_y = np.delete(T_prime_y,[0])
            idx.append(0)

    T_len -= K

# perfrom last QBC operation on the entire training dataset
knn_clf = KNeighborsClassifier(n_neighbors = 1)
knn_clf.fit(X_train.todense(),y_train)
nb_clf = MultinomialNB()
nb_clf.fit(X_train.todense(),y_train)
tree_clf = tree.DecisionTreeClassifier(max_depth = 30)
tree_clf.fit(X_train.todense(),y_train)

knn_pred = knn_clf.predict(X_test.todense())
nb_pred = nb_clf.predict(X_test.todense())
tree_pred = tree_clf.predict(X_test.todense())

train_size.append(len(X_train.todense()))
print(len(X_train.todense()))

majority_pred = []

for i in range(len(y_test)):
    votes = np.array([knn_pred[i],nb_pred[i],tree_pred[i]])
    majority_pred.append(np.argmax(np.bincount(votes)))

error = 1- accuracy_score(y_test, majority_pred)
test_errors.append(error)

# In[324]:

fig, ax = plt.subplots()
ax.plot(train_size,test_errors)

ax.set(xlabel='training set size', ylabel='test error',
        title='QBC')
annot_min(train_size,test_errors)

plt.show()


# This example is a simplistic "puzzle" in which the objective is to fill an nxn grid
# such that each row and column contains exactly 1 instance of each of the numbers 1-n.
# It is a demonstration of how to code a puzzle solver using a generic BFS framework.

from copy import deepcopy

from helper import permute
from bfs import *

class UF1problem(object):
	def __init__(self, initial, size, goal=None):
		# These values are specific to the problem you are solving (i.e. sudoku, nQueens, ...)
		self.size = size
		self.initial = initial
		self.goal = goal

		# For this example, an action is the assignment of an entire row
		# thus, the set of legal actions for all states is all permutations of viable numbers
		# Example: [[1,2,3],[1,3,2],[2,1,3],...]
		self.actions = permute( [ i for i in range(1,size+1) ] )

	def getActions( self, state ) :
		# Because all actions are legal for this example, they were generated once
		# in init, then are passed along here.
		# It might be that you generate only legal actions here, OR you generate all
		# actions, then test for legality in applyAction()
		if ( self.size == len(state) ):
			return []
		return self.actions

	def applyAction ( self, state, action ) :
		# It is very important that you generate a new variable with deepcopy for the new state
		# This code is problem specific. An action is applied by adding the configured
		# row (a permutation of numbers) to the state.
		newState = deepcopy(state)
		newState.append(action)
		return newState

	def isGoal ( self, state ):
		# Again, problem specific code.

		# If the state is empty, not done yet
		if not state:
			return False

		# If all rows have not yet been filled, not done yet
		if len(state) < self.size:
			return False

		# We have a completely filled in board. Check if there are any
		# duplicates across columns (our actions were defined such that duplicates
		# never appear in a given row)
		for col in range(self.size):
			if len( list(set([ row[col] for row in state ] ) )) < self.size :
				#print('State ',state,' not goal')
				return False

		# If we got here, the board is complete and legal
		print('WINNER')
		for i in range(self.size):
			boardrow = ''
			for j in range(self.size):
				boardrow += '   '+str(state[i][j])
			print(boardrow)
		return True

if __name__ == "__main__" :
	BFS(UF1problem([],4))

def testv1():
	# This is testing code. Currently it only tests 2 methods, but ideally, there should be a test
	# for each method. Many IDEs provide a test framework that makes this a lot easier,
	# but you can always create your own

	n = 3
	p = UF1problem([],n)

	# ------  TESTING getActions() --------------------
	answer = permute( [ i for i in range(1,n+1) ] )
	input = [ [], [[1,2,3]]]
	for i in input:
		a = p.getActions(i)
		msg = 'getActions('+str(i)+') ='+str(a)+' Expected '+str(answer)
		if not a == answer:
			msg = '**** FAIL **** : '+msg
		else:
			msg = ' pass : '+msg
		print(msg)

	# ------  TESTING applyActions() for 3x3 board  --------------------
	if 3 == n :
		io = [ [ [], [1,2,3], [[1,2,3]] ],
			   [ [[1,2,3]], [1,2,3], [[1,2,3],[1,2,3]] ],
			   [ [[1,2,3],[1,2,3]], [1,2,3], [[1,2,3],[1,2,3],[1,2,3]] ]
			   ]
		for i in io:
			a = p.applyAction( i[0], i[1] )
			msg = 'applyActions('+str(i[0])+','+str(i[1])+') = '+str(a)+' Expected '+str(i[2])
			if not a == i[2]:
				msg = '**** FAIL **** : '+msg
			else:
				msg = ' pass : '+msg
			print(msg)
#testv1()


import FWCore.ParameterSet.Config as cms

# Select MC truth
from L1TriggerOffline.L1Analyzer.GenSelection_cff import *
# Select L1
from L1TriggerOffline.L1Analyzer.L1Selection_cff import *
# Histogram limits
from L1TriggerOffline.L1Analyzer.HistoLimits_cfi import *
# Root output file
from L1TriggerOffline.L1Analyzer.TFile_cfi import *
# Match generator taus and L1 tau jets
MatchTauJetsMc = cms.EDFilter("TrivialDeltaRMatcher",
    src = cms.InputTag("SelectL1TauJets"),
    distMin = cms.double(0.5),
    matched = cms.InputTag("SelectGenTauJets")
)

# Match L1 and generator tau jets
MatchMcTauJets = cms.EDFilter("TrivialDeltaRMatcher",
    src = cms.InputTag("SelectGenTauJets"),
    distMin = cms.double(0.5),
    matched = cms.InputTag("SelectL1TauJets")
)

# Analyzer
L1AnalyzerTauJetsMC = cms.EDAnalyzer("L1Analyzer",
    histoLimits,
    EffMatchMapSource = cms.untracked.InputTag("MatchMcTauJets"),
    ReferenceSource = cms.untracked.InputTag("SelectGenTauJets"),
    CandidateSource = cms.untracked.InputTag("SelectL1TauJets"),
    ResMatchMapSource = cms.untracked.InputTag("MatchTauJetsMc")
)

# Define analysis sequence
L1TauJetMCAnalysis = cms.Sequence(L1TauJetSelection+GenTauJetSelection*MatchTauJetsMc+MatchMcTauJets*L1AnalyzerTauJetsMC)


import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
from matplotlib.figure import Figure
import pandas as pd
import cv2
import pickle
from flask import Response
import io
import base64

def run(input_img):

    sess = tf.Session()
    # img = cv2.imread(img_path,cv2.IMREAD_GRAYSCALE)
    img = np.float32(input_img)
    img = cv2.resize(img, (224, 224))
    img = img/127.5 - 1
    img = tf.convert_to_tensor(img)
    img = tf.reshape(img,(1,224,224,1))
    i = sess.run(tf.squeeze(img))

    def c2c(hm,n=5):
        ind = hm.argsort(axis=None)[-n:]
        topind = np.unravel_index(ind,hm.shape)

        i0,i1,hsum = 0,0,0
        for indx in zip(topind[0],topind[1]):
            h = hm[indx[0],indx[1]]
            hsum += h
            i0 += indx[0]*h
            i1 += indx[1]*h

        i0 /= hsum
        i1 /= hsum
        return [i1,i0]

    def show(a, b):

        #save a plot figure as png
        #ax = plt.subplot('111')
        #plt.xlabel('X Axis', axes=ax)
        #plt.ylabel('Y Axis', axes=ax)
        #ax.imshow(a)
        #ax.scatter(b[:, 0], b[:, 1], s=20, marker='.', c='m')
        #plt.savefig('test.png', bbox_inches="tight")

        fig = Figure(figsize=(3, 3))
        canvas = FigureCanvas(fig)
        ax = fig.add_subplot(111)
        ax.imshow(a)
        ax.scatter(b[:, 0], b[:, 1], s=20, marker='.', c='m')
        ax.axis('off')

        canvas.draw()
        buf = fig.canvas.tostring_rgb()
        ncol, nrows = fig.canvas.get_width_height()
        return np.fromstring(buf, dtype=np.uint8).reshape((ncol,nrows,3))

    def show_plot(a,b):
        plt.imshow(a)
        plt.scatter(b[:, 0], b[:, 1], s=20, marker='.', c='m')

    def cd(hm):
        assert len(hm.shape) == 3
        Nlandmarks = hm.shape[-1]
        est_xy = []
        for i in range(Nlandmarks):
            hmi = hm[:,:,i]
            est_xy.append(c2c(hmi))

        return np.array(est_xy)

    with tf.variable_scope('conv1') as scope:
        m = tf.nn.conv2d(img,tf.get_variable(name = 'filter',shape = [3,3,1,16]),strides =(1,1,1,1),padding ="SAME") + tf.get_variable(name = 'bias', shape = [16])

    with tf.variable_scope('bn1') as scope:
        m = tf.nn.relu(tf.layers.batch_normalization(inputs = m,axis = -1,momentum = 0.9,epsilon = 0.001,center = True,scale = True,
                                                        training = False))

    with tf.variable_scope('conv2') as scope:
        m = tf.nn.conv2d(m,tf.get_variable(name = 'filter',shape = [3,3,16,64]),strides =(1,1,1,1),padding ="SAME") + tf.get_variable(name = 'bias', shape = [64])

    with tf.variable_scope('bn2') as scope:
        m = tf.nn.relu(tf.layers.batch_normalization(inputs = m,axis = -1,momentum = 0.9,epsilon = 0.001,center = True,scale = True,
                                                        training = False))

    with tf.variable_scope('conv3') as scope:
        m = tf.nn.conv2d(m,tf.get_variable(name = 'filter',shape = [3,3,64,64]),strides =(1,2,2,1),padding ="SAME") + tf.get_variable(name = 'bias', shape = [64])

    with tf.variable_scope('bn3') as scope:
        m = tf.nn.relu(tf.layers.batch_normalization(inputs = m,axis = -1,momentum = 0.9,epsilon = 0.001,center = True,scale = True,
                                                        training = False))

    with tf.variable_scope('conv4') as scope:
        m = tf.nn.conv2d(m,tf.get_variable(name = 'filter',shape = [3,3,64,128]),strides =(1,2,2,1),padding ="SAME") + tf.get_variable(name = 'bias', shape = [128])

    with tf.variable_scope('bn4') as scope:
        m = tf.nn.relu(tf.layers.batch_normalization(inputs = m,axis = -1,momentum = 0.9,epsilon = 0.001,center = True,scale = True,
                                                        training = False))

    with tf.variable_scope('conv5') as scope:
        m = tf.nn.conv2d(m,tf.get_variable(name = 'filter',shape = [3,3,128,160]),strides =(1,1,1,1),padding ="SAME") + tf.get_variable(name = 'bias', shape = [160])

    with tf.variable_scope('bn5') as scope:
        m = tf.nn.relu(tf.layers.batch_normalization(inputs = m,axis = -1,momentum = 0.9,epsilon = 0.001,center = True,scale = True,
                                                        training = False))

    with tf.variable_scope('bottleneck') as scope:
        m = tf.nn.conv2d(m,tf.get_variable(name = 'filter',shape = [7,7,160,160]),strides =(1,1,1,1),padding ="SAME") + tf.get_variable(name = 'bias', shape = [160])

    with tf.variable_scope('bn6') as scope:
        m = tf.nn.relu(tf.layers.batch_normalization(inputs = m,axis = -1,momentum = 0.9,epsilon = 0.001,center = True,scale = True,
                                                        training = False))

    with tf.variable_scope('dconv6') as scope:
        pred = tf.nn.conv2d_transpose(value = m,filter = tf.get_variable(name = 'filter',shape=[4,4,68,160]),output_shape = [1,224,224,68],strides = (1,4,4,1))+tf.get_variable(name='bias',shape = [68])

    saver = tf.train.Saver()
    saver.restore(sess,"face_pt_testing_heat_map1.ckpt")
    hm = sess.run(tf.squeeze(pred))
    out = cd(hm)
    np.save("result.npy",out)

    #plt.figure(figsize=(5, 5))
    response = show(i, out)
    #show_plot(i,out)
    #plt.show()

    return response

# if __name__ == "__main__":
#     res = run("test_image1.jpeg")
#     print(repr(res))


# --------------------------------------------------------------------------
# Version 12.9.0
# --------------------------------------------------------------------------
# Licensed Materials - Property of IBM
# 5725-A06 5725-A29 5724-Y48 5724-Y49 5724-Y54 5724-Y55 5655-Y21
# Copyright IBM Corporation 2000, 2019. All Rights Reserved.
#
# US Government Users Restricted Rights - Use, duplication or
# disclosure restricted by GSA ADP Schedule Contract with
# IBM Corp.
# --------------------------------------------------------------------------
"""

"""

from ._parameters_auto import *
from ._parameter_classes import *

def barrier_limits_members(env, parent):
    return dict(_name = "limits",
                help = lambda : "Limits for barrier optimization.",
                corrections = NumParameter(env, BarrierLimitsCorrections, parent, 'corrections'),
                growth = NumParameter(env, BarrierLimitsGrowth, parent, 'growth'),
                iteration = NumParameter(env, BarrierLimitsIteration, parent, 'iteration'),
                objrange = NumParameter(env, BarrierLimitsObjRange, parent, 'objrange'),
                )

def barrier_members(env, parent):
    return dict(_name = "barrier",
                help = lambda : "Parameters for barrier optimization.",
                algorithm = NumParameter(env, BarrierAlgorithm, parent, 'algorithm', bar_alg_constants),
                colnonzeros = NumParameter(env, BarrierColNonzeros, parent, 'colnonzeros'),
                convergetol = NumParameter(env, BarrierConvergeTol, parent, 'convergetol'),
                crossover = NumParameter(env, BarrierCrossover, parent, 'crossover', crossover_constants),
                display = NumParameter(env, BarrierDisplay, parent, 'display', display_constants),
                limits = ParameterGroup(env, barrier_limits_members, parent),
                ordering = NumParameter(env, BarrierOrdering, parent, 'ordering', bar_order_constants),
                qcpconvergetol = NumParameter(env, BarrierQCPConvergeTol, parent, 'qcpconvergetol'),
                startalg = NumParameter(env, BarrierStartAlg, parent, 'startalg', bar_start_alg_constants),
                )

def benders_tolerances_members(env, parent):
    return dict(_name = "tolerances",
                help = lambda : "Numerical tolerances for Benders cuts.",
                feasibilitycut = NumParameter(env, BendersTolerancesfeasibilitycut, parent, 'feasibilitycut'),
                optimalitycut = NumParameter(env, BendersTolerancesoptimalitycut, parent, 'optimalitycut'),
                )

def benders_members(env, parent):
    return dict(_name = "benders",
                help = lambda : "Parameters for benders optimization.",
                strategy = NumParameter(env, BendersStrategy, parent, 'strategy', benders_strategy_constants),
                tolerances = ParameterGroup(env, benders_tolerances_members, parent),
                workeralgorithm = NumParameter(env, BendersWorkerAlgorithm, parent, 'workeralgorithm', subalg_constants),
                )

def conflict_members(env, parent):
    return dict(_name = "conflict",
                help = lambda : "Parameters for finding conflicts.",
                algorithm = NumParameter(env, ConflictAlgorithm, parent, 'algorithm', conflict_algorithm_constants),
                display = NumParameter(env, ConflictDisplay, parent, 'display', display_constants),
                )

def distmip_rampup_members(env, parent):
    return dict(_name = "rampup",
                help = lambda : "Rampup related parameters in distributed MIP.",
                dettimelimit = NumParameter(env, DistMIPRampupDetTimeLimit, parent, 'dettimelimit'),
                duration = NumParameter(env, DistMIPRampupDuration, parent, 'duration', rampup_duration_constants),
                timelimit = NumParameter(env, DistMIPRampupTimeLimit, parent, 'timelimit'),
                )

def distmip_members(env, parent):
    return dict(_name = "distmip",
                help = lambda : "Distributed parallel mixed integer optimization.",
                rampup = ParameterGroup(env, distmip_rampup_members, parent),
                )

def emphasis_members(env, parent):
    return dict(_name = "emphasis",
                help = lambda : "Optimization emphasis.",
                memory = NumParameter(env, EmphasisMemory, parent, 'memory', off_on_constants),
                mip = NumParameter(env, EmphasisMIP, parent, 'mip', mip_emph_constants),
                numerical = NumParameter(env, EmphasisNumerical, parent, 'numerical', off_on_constants),
                )

def feasopt_members(env, parent):
    return dict(_name = "feasopt",
                help = lambda : "Parameters for feasopt.",
                mode = NumParameter(env, FeasoptMode, parent, 'mode', feasopt_mode_constants),
                tolerance = NumParameter(env, FeasoptTolerance, parent, 'tolerance'),
                )

def mip_cuts_members(env, parent):
    return dict(_name = "cuts",
                help = lambda : "Types of cuts used during mixed integer optimization.",
                bqp = NumParameter(env, MIPCutsBQP, parent, 'bqp', v_agg_constants),
                cliques = NumParameter(env, MIPCutsCliques, parent, 'cliques', v_agg_constants),
                covers = NumParameter(env, MIPCutsCovers, parent, 'covers', v_agg_constants),
                disjunctive = NumParameter(env, MIPCutsDisjunctive, parent, 'disjunctive', v_agg_constants),
                flowcovers = NumParameter(env, MIPCutsFlowCovers, parent, 'flowcovers', agg_constants),
                gomory = NumParameter(env, MIPCutsGomory, parent, 'gomory', agg_constants),
                gubcovers = NumParameter(env, MIPCutsGUBCovers, parent, 'gubcovers', agg_constants),
                implied = NumParameter(env, MIPCutsImplied, parent, 'implied', agg_constants),
                liftproj = NumParameter(env, MIPCutsLiftProj, parent, 'liftproj', v_agg_constants),
                localimplied = NumParameter(env, MIPCutsLocalImplied, parent, 'localimplied', v_agg_constants),
                mcfcut = NumParameter(env, MIPCutsMCFCut, parent, 'mcfcut', agg_constants),
                mircut = NumParameter(env, MIPCutsMIRCut, parent, 'mircut', agg_constants),
                pathcut = NumParameter(env, MIPCutsPathCut, parent, 'pathcut', agg_constants),
                rlt = NumParameter(env, MIPCutsRLT, parent, 'rlt', v_agg_constants),
                zerohalfcut = NumParameter(env, MIPCutsZeroHalfCut, parent, 'zerohalfcut', agg_constants),
                )

def mip_limits_members(env, parent):
    return dict(_name = "limits",
                help = lambda : "Limits for mixed integer optimization.",
                aggforcut = NumParameter(env, MIPLimitsAggForCut, parent, 'aggforcut'),
                auxrootthreads = NumParameter(env, MIPLimitsAuxRootThreads, parent, 'auxrootthreads'),
                cutpasses = NumParameter(env, MIPLimitsCutPasses, parent, 'cutpasses'),
                cutsfactor = NumParameter(env, MIPLimitsCutsFactor, parent, 'cutsfactor'),
                eachcutlimit = NumParameter(env, MIPLimitsEachCutLimit, parent, 'eachcutlimit'),
                gomorycand = NumParameter(env, MIPLimitsGomoryCand, parent, 'gomorycand'),
                gomorypass = NumParameter(env, MIPLimitsGomoryPass, parent, 'gomorypass'),
                nodes = NumParameter(env, MIPLimitsNodes, parent, 'nodes'),
                polishtime = NumParameter(env, MIPLimitsPolishTime, parent, 'polishtime'),
                populate = NumParameter(env, MIPLimitsPopulate, parent, 'populate'),
                probedettime = NumParameter(env, MIPLimitsProbeDetTime, parent, 'probedettime'),
                probetime = NumParameter(env, MIPLimitsProbeTime, parent, 'probetime'),
                repairtries = NumParameter(env, MIPLimitsRepairTries, parent, 'repairtries'),
                solutions = NumParameter(env, MIPLimitsSolutions, parent, 'solutions'),
                strongcand = NumParameter(env, MIPLimitsStrongCand, parent, 'strongcand'),
                strongit = NumParameter(env, MIPLimitsStrongIt, parent, 'strongit'),
                treememory = NumParameter(env, MIPLimitsTreeMemory, parent, 'treememory'),
                )

def mip_polishafter_members(env, parent):
    return dict(_name = "polishafter",
                help = lambda : "Starting conditions for solution polishing.",
                absmipgap = NumParameter(env, MIPPolishAfterAbsMIPGap, parent, 'absmipgap'),
                dettime = NumParameter(env, MIPPolishAfterDetTime, parent, 'dettime'),
                mipgap = NumParameter(env, MIPPolishAfterMIPGap, parent, 'mipgap'),
                nodes = NumParameter(env, MIPPolishAfterNodes, parent, 'nodes'),
                solutions = NumParameter(env, MIPPolishAfterSolutions, parent, 'solutions'),
                time = NumParameter(env, MIPPolishAfterTime, parent, 'time'),
                )

def mip_pool_members(env, parent):
    return dict(_name = "pool",
                help = lambda : "Solution pool characteristics.",
                absgap = NumParameter(env, MIPPoolAbsGap, parent, 'absgap'),
                capacity = NumParameter(env, MIPPoolCapacity, parent, 'capacity'),
                intensity = NumParameter(env, MIPPoolIntensity, parent, 'intensity', v_agg_constants),
                relgap = NumParameter(env, MIPPoolRelGap, parent, 'relgap'),
                replace = NumParameter(env, MIPPoolReplace, parent, 'replace', replace_constants),
                )

def mip_strategy_members(env, parent):
    return dict(_name = "strategy",
                help = lambda : "Strategy for mixed integer optimization.",
                backtrack = NumParameter(env, MIPStrategyBacktrack, parent, 'backtrack'),
                bbinterval = NumParameter(env, MIPStrategyBBInterval, parent, 'bbinterval'),
                branch = NumParameter(env, MIPStrategyBranch, parent, 'branch', brdir_constants),
                dive = NumParameter(env, MIPStrategyDive, parent, 'dive', dive_constants),
                file = NumParameter(env, MIPStrategyFile, parent, 'file', file_constants),
                fpheur = NumParameter(env, MIPStrategyFPHeur, parent, 'fpheur', fpheur_constants),
                heuristicfreq = NumParameter(env, MIPStrategyHeuristicFreq, parent, 'heuristicfreq'),
                kappastats = NumParameter(env, MIPStrategyKappaStats, parent, 'kappastats', kappastats_constants),
                lbheur = NumParameter(env, MIPStrategyLBHeur, parent, 'lbheur', off_on_constants),
                miqcpstrat = NumParameter(env, MIPStrategyMIQCPStrat, parent, 'miqcpstrat', miqcp_constants),
                nodeselect = NumParameter(env, MIPStrategyNodeSelect, parent, 'nodeselect', nodesel_constants),
                order = NumParameter(env, MIPStrategyOrder, parent, 'order', off_on_constants),
                presolvenode = NumParameter(env, MIPStrategyPresolveNode, parent, 'presolvenode', presolve_constants),
                probe = NumParameter(env, MIPStrategyProbe, parent, 'probe', v_agg_constants),
                rinsheur = NumParameter(env, MIPStrategyRINSHeur, parent, 'rinsheur'),
                search = NumParameter(env, MIPStrategySearch, parent, 'search', search_constants),
                startalgorithm = NumParameter(env, MIPStrategyStartAlgorithm, parent, 'startalgorithm', alg_constants),
                subalgorithm = NumParameter(env, MIPStrategySubAlgorithm, parent, 'subalgorithm', subalg_constants),
                variableselect = NumParameter(env, MIPStrategyVariableSelect, parent, 'variableselect', varsel_constants),
                )

def mip_submip_members(env, parent):
    return dict(_name = "submip",
                help = lambda : "Parameters used when solving sub-MIPs.",
                startalg = NumParameter(env, MIPSubMIPStartAlg, parent, 'startalg', subalg_constants),
                subalg = NumParameter(env, MIPSubMIPSubAlg, parent, 'subalg', subalg_constants),
                nodelimit = NumParameter(env, MIPSubMIPNodeLimit, parent, 'nodelimit'),
                scale = NumParameter(env, MIPSubMIPScale, parent, 'scale', scale_constants),
                )

def mip_tolerances_members(env, parent):
    return dict(_name = "tolerances",
                help = lambda : "Tolerances for mixed integer optimization.",
                absmipgap = NumParameter(env, MIPTolerancesAbsMIPGap, parent, 'absmipgap'),
                integrality = NumParameter(env, MIPTolerancesIntegrality, parent, 'integrality'),
                lowercutoff = NumParameter(env, MIPTolerancesLowerCutoff, parent, 'lowercutoff'),
                mipgap = NumParameter(env, MIPTolerancesMIPGap, parent, 'mipgap'),
                objdifference = NumParameter(env, MIPTolerancesObjDifference, parent, 'objdifference'),
                relobjdifference = NumParameter(env, MIPTolerancesRelObjDifference, parent, 'relobjdifference'),
                uppercutoff = NumParameter(env, MIPTolerancesUpperCutoff, parent, 'uppercutoff'),
                )

def mip_members(env, parent):
    return dict(_name = "mip",
                help = lambda : "Parameters for mixed integer optimization.",
                cuts = ParameterGroup(env, mip_cuts_members, parent),
                display = NumParameter(env, MIPDisplay, parent, 'display', mip_display_constants),
                interval = NumParameter(env, MIPInterval, parent, 'interval'),
                limits = ParameterGroup(env, mip_limits_members, parent),
                ordertype = NumParameter(env, MIPOrderType, parent, 'ordertype', ordertype_constants),
                polishafter = ParameterGroup(env, mip_polishafter_members, parent),
                pool = ParameterGroup(env, mip_pool_members, parent),
                strategy = ParameterGroup(env, mip_strategy_members, parent),
                submip = ParameterGroup(env, mip_submip_members, parent),
                tolerances = ParameterGroup(env, mip_tolerances_members, parent),
                )

def multiobjective_members(env, parent):
    return dict(_name = "multiobjective",
                help = lambda : "Parameters for multi-objective optimization.",
                display = NumParameter(env, MultiObjectiveDisplay, parent, 'display', display_constants),
                )

def network_tolerances_members(env, parent):
    return dict(_name = "tolerances",
                help = lambda : "Numerical tolerances for network simplex optimization.",
                feasibility = NumParameter(env, NetworkTolerancesFeasibility, parent, 'feasibility'),
                optimality = NumParameter(env, NetworkTolerancesOptimality, parent, 'optimality'),
                )

def network_members(env, parent):
    return dict(_name = "network",
                help = lambda : "Parameters for network optimizations.",
                display = NumParameter(env, NetworkDisplay, parent, 'display', network_display_constants),
                iterations = NumParameter(env, NetworkIterations, parent, 'iterations'),
                netfind = NumParameter(env, NetworkNetFind, parent, 'netfind', network_netfind_constants),
                pricing = NumParameter(env, NetworkPricing, parent, 'pricing', network_pricing_constants),
                tolerances = ParameterGroup(env, network_tolerances_members, parent),
                )

def output_members(env, parent):
    return dict(_name = "output",
                help = lambda : "Extent and destinations of outputs.",
                clonelog = NumParameter(env, OutputCloneLog, parent, 'clonelog', off_on_constants),
                intsolfileprefix = StrParameter(env, OutputIntSolFilePrefix, parent, 'intsolfileprefix'),
                mpslong = NumParameter(env, OutputMPSLong, parent, 'mpslong', off_on_constants),
                writelevel = NumParameter(env, OutputWriteLevel, parent, 'writelevel', writelevel_constants),
                )

def preprocessing_members(env, parent):
    return dict(_name = "preprocessing",
                help = lambda : "Parameters for preprocessing.",
                aggregator = NumParameter(env, PreprocessingAggregator, parent, 'aggregator'),
                boundstrength = NumParameter(env, PreprocessingBoundStrength, parent, 'boundstrength', auto_off_on_constants),
                coeffreduce = NumParameter(env, PreprocessingCoeffReduce, parent, 'coeffreduce', coeffreduce_constants),
                dependency = NumParameter(env, PreprocessingDependency, parent, 'dependency', dependency_constants),
                dual = NumParameter(env, PreprocessingDual, parent, 'dual', dual_constants),
                fill = NumParameter(env, PreprocessingFill, parent, 'fill'),
                folding = NumParameter(env, PreprocessingFolding, parent, 'folding'),
                linear = NumParameter(env, PreprocessingLinear, parent, 'linear', linear_constants),
                numpass = NumParameter(env, PreprocessingNumPass, parent, 'numpass'),
                presolve = NumParameter(env, PreprocessingPresolve, parent, 'presolve', off_on_constants),
                qcpduals = NumParameter(env, PreprocessingQCPDuals, parent, 'qcpduals', qcpduals_constants),
                qpmakepsd = NumParameter(env, PreprocessingQPMakePSD, parent, 'qpmakepsd', off_on_constants),
                qtolin = NumParameter(env, PreprocessingQToLin, parent, 'qtolin', auto_off_on_constants),
                reduce = NumParameter(env, PreprocessingReduce, parent, 'reduce', prered_constants),
                relax = NumParameter(env, PreprocessingRelax, parent, 'relax', auto_off_on_constants),
                repeatpresolve = NumParameter(env, PreprocessingRepeatPresolve, parent, 'repeatpresolve', repeatpre_constants),
                symmetry = NumParameter(env, PreprocessingSymmetry, parent, 'symmetry', sym_constants),
                )

def read_members(env, parent):
    return dict(_name = "read",
                help = lambda : "Problem read parameters.",
                apiencoding = StrParameter(env, ReadAPIEncoding, parent, 'apiencoding'),
                constraints = NumParameter(env, ReadConstraints, parent, 'constraints'),
                datacheck = NumParameter(env, ReadDataCheck, parent, 'datacheck', datacheck_constants),
                fileencoding = StrParameter(env, ReadFileEncoding, parent, 'fileencoding'),
                nonzeros = NumParameter(env, ReadNonzeros, parent, 'nonzeros'),
                qpnonzeros = NumParameter(env, ReadQPNonzeros, parent, 'qpnonzeros'),
                scale = NumParameter(env, ReadScale, parent, 'scale', scale_constants),
                variables = NumParameter(env, ReadVariables, parent, 'variables'),
                warninglimit = NumParameter(env, ReadWarningLimit, parent, 'warninglimit'),
                )

def sifting_members(env, parent):
    return dict(_name = "sifting",
                help = lambda : "Parameters for sifting optimization.",
                algorithm = NumParameter(env, SiftingAlgorithm, parent, 'algorithm', sift_alg_constants),
                simplex = NumParameter(env, SiftingSimplex, parent, 'simplex', off_on_constants),
                display = NumParameter(env, SiftingDisplay, parent, 'display', display_constants),
                iterations = NumParameter(env, SiftingIterations, parent, 'iterations'),
                )

def simplex_limits_members(env, parent):
    return dict(_name = "limits",
                help = lambda : "Limits for simplex optimization.",
                iterations = NumParameter(env, SimplexLimitsIterations, parent, 'iterations'),
                lowerobj = NumParameter(env, SimplexLimitsLowerObj, parent, 'lowerobj'),
                perturbation = NumParameter(env, SimplexLimitsPerturbation, parent, 'perturbation'),
                singularity = NumParameter(env, SimplexLimitsSingularity, parent, 'singularity'),
                upperobj = NumParameter(env, SimplexLimitsUpperObj, parent, 'upperobj'),
                )

def simplex_perturbation_members(env, parent):
    return dict(_name = "perturbation",
                help = lambda : "Perturbation controls.",
                constant = NumParameter(env, SimplexPerturbationConstant, parent, 'constant'),
                indicator = NumParameter(env, SimplexPerturbationIndicator, parent, 'indicator', off_on_constants),
                )

def simplex_tolerances_members(env, parent):
    return dict(_name = "tolerances",
                help = lambda : "Numerical tolerances for simplex optimization.",
                feasibility = NumParameter(env, SimplexTolerancesFeasibility, parent, 'feasibility'),
                markowitz = NumParameter(env, SimplexTolerancesMarkowitz, parent, 'markowitz'),
                optimality = NumParameter(env, SimplexTolerancesOptimality, parent, 'optimality'),
                )

def simplex_members(env, parent):
    return dict(_name = "simplex",
                help = lambda : "Parameters for primal and dual simplex optimizations.",
                crash = NumParameter(env, SimplexCrash, parent, 'crash'),
                dgradient = NumParameter(env, SimplexDGradient, parent, 'dgradient', dual_pricing_constants),
                display = NumParameter(env, SimplexDisplay, parent, 'display', display_constants),
                dynamicrows = NumParameter(env, SimplexDynamicRows, parent, 'dynamicrows'),
                limits = ParameterGroup(env, simplex_limits_members, parent),
                perturbation = ParameterGroup(env, simplex_perturbation_members, parent),
                pgradient = NumParameter(env, SimplexPGradient, parent, 'pgradient', primal_pricing_constants),
                pricing = NumParameter(env, SimplexPricing, parent, 'pricing'),
                refactor = NumParameter(env, SimplexRefactor, parent, 'refactor'),
                tolerances = ParameterGroup(env, simplex_tolerances_members, parent),
                )

def tune_members(env, parent):
    return dict(_name = "tune",
                help = lambda : "Parameters for parameter tuning.",
                dettimelimit = NumParameter(env, TuneDetTimeLimit, parent, 'dettimelimit'),
                display = NumParameter(env, TuneDisplay, parent, 'display', tune_display_constants),
                measure = NumParameter(env, TuneMeasure, parent, 'measure', measure_constants),
                repeat = NumParameter(env, TuneRepeat, parent, 'repeat'),
                timelimit = NumParameter(env, TuneTimeLimit, parent, 'timelimit'),
                )

def root_members(env, parent):
    return dict(_name = "parameters",
                help = lambda : "CPLEX parameter hierarchy.",
                advance = NumParameter(env, setAdvance, parent, 'advance', advance_constants),
                barrier = ParameterGroup(env, barrier_members, parent),
                benders = ParameterGroup(env, benders_members, parent),
                clocktype = NumParameter(env, setClockType, parent, 'clocktype', clocktype_constants),
                conflict = ParameterGroup(env, conflict_members, parent),
                cpumask = StrParameter(env, setCPUmask, parent, 'cpumask'),
                dettimelimit = NumParameter(env, setDetTimeLimit, parent, 'dettimelimit'),
                distmip = ParameterGroup(env, distmip_members, parent),
                emphasis = ParameterGroup(env, emphasis_members, parent),
                feasopt = ParameterGroup(env, feasopt_members, parent),
                lpmethod = NumParameter(env, setLPMethod, parent, 'lpmethod', alg_constants),
                mip = ParameterGroup(env, mip_members, parent),
                multiobjective = ParameterGroup(env, multiobjective_members, parent),
                network = ParameterGroup(env, network_members, parent),
                optimalitytarget = NumParameter(env, setOptimalityTarget, parent, 'optimalitytarget', optimalitytarget_constants),
                output = ParameterGroup(env, output_members, parent),
                parallel = NumParameter(env, setParallel, parent, 'parallel', par_constants),
                paramdisplay = NumParameter(env, setParamDisplay, parent, 'paramdisplay', off_on_constants),
                preprocessing = ParameterGroup(env, preprocessing_members, parent),
                qpmethod = NumParameter(env, setQPMethod, parent, 'qpmethod', qp_alg_constants),
                randomseed = NumParameter(env, setRandomSeed, parent, 'randomseed'),
                read = ParameterGroup(env, read_members, parent),
                record = NumParameter(env, setRecord, parent, 'record', off_on_constants),
                sifting = ParameterGroup(env, sifting_members, parent),
                simplex = ParameterGroup(env, simplex_members, parent),
                solutiontype = NumParameter(env, setSolutionType, parent, 'solutiontype', solutiontype_constants),
                threads = NumParameter(env, setThreads, parent, 'threads'),
                timelimit = NumParameter(env, setTimeLimit, parent, 'timelimit'),
                tune = ParameterGroup(env, tune_members, parent),
                workdir = StrParameter(env, setWorkDir, parent, 'workdir'),
                workmem = NumParameter(env, setWorkMem, parent, 'workmem'),
                )


# -*- coding: utf-8 -*-
"""
Generic Constants and Classes
"""
from __future__ import generator_stop

import sys

SEPARATOR =  "\r\n\r\n"
SEPARATOR_BYTES = SEPARATOR.encode("utf-8")
DID_LENGTH = 52

PROPAGATION_DELAY = 60.0  # network propagation time for consensus

ANON_EXPIRATION_DELAY = 60 * 60 * 24  # 24 hours in units of seconds
#ANON_EXPIRATION_DELAY =  10  # ten seconds for testing

fakeHidKind = False  # module global flag, If True enable fake hid kind that does not require validation

class BluepeaError(Exception):
    """
    Base Class for bluepea exceptions

    To use   raise BluepeaError("Error: message")
    """

class ValidationError(BluepeaError):
    """
    Validation related errors
    Usage:
        raise ValidationError("error message")
    """


from django.conf.urls import patterns, include, url
from django.conf import settings
from django.views.generic.simple import direct_to_template

# Uncomment the next two lines to enable the admin:
from django.contrib import admin
admin.autodiscover()

urlpatterns = patterns('',
    url(r'^$', 'publicweb.views.Home'),
    url(r'^generator$', 'generator.views.Generator'),
    url(r'^graveyards$', 'publicweb.views.GraveyardsList'),
    url(r'^graveyard-(?P<g_id>\d+)$', 'publicweb.views.GraveyardShow'),
    url(r'^place-(?P<p_id>\d+)$', 'publicweb.views.PlaceShow'),
    url(r'^place-(?P<p_id>\d+)/client-(?P<c_id>\d+)$', 'publicweb.views.ClientShow'),
    url(r'^api$', 'publicweb.views.ApiDocs'),
	url(r'^xmlrpc_endpoint$', 'publicweb.xmlrpc.rpc_handler'),

    # Uncomment the admin/doc line below to enable admin documentation:
    # url(r'^admin/doc/', include('django.contrib.admindocs.urls')),

    # Uncomment the next line to enable the admin:
    url(r'^admin/', include(admin.site.urls)),
	url(r'^static/(?P<path>.*)$', 'django.views.static.serve', {'document_root': settings.STATIC_ROOT}),
)


# Copyright (C) 2020 Jørgen S. Dokken
#
# This file is part of DOLFINX_MPC
#
# SPDX-License-Identifier:    LGPL-3.0-or-later

import typing
import numba
from petsc4py import PETSc
import types
from dolfinx import function, fem, MPI
from .assemble_matrix import in_numpy_array, add_diagonal
import numpy

def backsubstitution(mpc, vector, dofmap):

    # Unravel data from MPC
    slave_cells = mpc.slave_cells()
    coefficients = mpc.coefficients()
    masters = mpc.masters_local()
    slave_cell_to_dofs = mpc.slave_cell_to_dofs()
    cell_to_slave = slave_cell_to_dofs.array()
    cell_to_slave_offset = slave_cell_to_dofs.offsets()
    slaves = mpc.slaves()
    masters_local = masters.array()
    offsets = masters.offsets()
    mpc_wrapper = (slaves, slave_cells, cell_to_slave, cell_to_slave_offset,
                   masters_local, coefficients, offsets)
    num_dofs_per_element = dofmap.dof_layout.num_dofs
    index_map = mpc.index_map()
    global_indices = index_map.indices(True)
    backsubstitution_numba(vector, dofmap.list.array(),
                           num_dofs_per_element, mpc_wrapper, global_indices)
    vector.ghostUpdate(addv=PETSc.InsertMode.INSERT,
                       mode=PETSc.ScatterMode.FORWARD)
    return vector

@numba.njit(cache=True)
def backsubstitution_numba(b, dofmap, num_dofs_per_element, mpc,
                           global_indices):
    """
    Insert mpc values into vector bc
    """
    (slaves, slave_cells, cell_to_slave, cell_to_slave_offset,
     masters_local, coefficients, offsets) = mpc
    slaves_visited = numpy.empty(0, dtype=numpy.float64)

    # Loop through slave cells
    for (index, cell_index) in enumerate(slave_cells):
        cell_slaves = cell_to_slave[cell_to_slave_offset[index]:
                                    cell_to_slave_offset[index+1]]
        local_dofs = dofmap[num_dofs_per_element * cell_index:
                            num_dofs_per_element * cell_index
                            + num_dofs_per_element]

        # Find the global index of the slaves on the cell in the slaves-array
        global_slaves_index = []
        for gi in range(len(slaves)):
            if in_numpy_array(cell_slaves, slaves[gi]):
                global_slaves_index.append(gi)

        for slave_index in global_slaves_index:
            slave = slaves[slave_index]
            k = -1
            # Find local position of slave dof
            for local_dof in local_dofs:
                if global_indices[local_dof] == slave:
                    k = local_dof
            assert k != -1
            # Check if we have already inserted for this slave
            if not in_numpy_array(slaves_visited, slave):
                slaves_visited = numpy.append(slaves_visited, slave)
                slaves_masters = masters_local[offsets[slave_index]:
                                               offsets[slave_index+1]]
                slaves_coeffs = coefficients[offsets[slave_index]:
                                             offsets[slave_index+1]]
                for (master, coeff) in zip(slaves_masters, slaves_coeffs):
                    b[k] += coeff*b[master]

def slave_master_structure(V: function.FunctionSpace, slave_master_dict:
                           typing.Dict[types.FunctionType,
                                       typing.Dict[
                                           types.FunctionType, float]],
                           subspace_slave=None,
                           subspace_master=None):
    """
    Returns the data structures required to build a multi-point constraint.
    Given a nested dictionary, where the first keys are functions for
    geometrically locating the slave degrees of freedom. The values of these
    keys are another dictionary, containing functions for geometrically
    locating the master degree of freedom. The value of the nested dictionary
    is the coefficient the master degree of freedom should be multiplied with
    in the multi point constraint.
    Example:
       If u0 = alpha u1 + beta u2, u3 = beta u4 + gamma u5
       slave_master_dict = {lambda x loc_u0:{lambda x loc_u1: alpha,
                                             lambda x loc_u2: beta},
                            lambda x loc_u3:{lambda x loc_u4: beta,
                                             lambda x loc_u5: gamma}}
    """
    slaves = []
    masters = []
    coeffs = []
    offsets = []
    local_min = (V.dofmap.index_map.local_range[0]
                 * V.dofmap.index_map.block_size)
    if subspace_slave is not None:
        Vsub_slave = V.sub(subspace_slave).collapse()
    if subspace_master is not None:
        Vsub_master = V.sub(subspace_master).collapse()
    for slave in slave_master_dict.keys():
        offsets.append(len(masters))
        if subspace_slave is None:
            dof = fem.locate_dofs_geometrical(V, slave) + local_min
            dof_global = numpy.vstack(MPI.comm_world.allgather(dof))[0]

        else:
            dof = fem.locate_dofs_geometrical((V.sub(subspace_slave),
                                               Vsub_slave),
                                              slave)
            for (i, d) in enumerate(dof):
                dof[i] += local_min
            dof_global = numpy.vstack(MPI.comm_world.allgather(dof))[0, 0]
        slaves.append(dof_global)
        for master in slave_master_dict[slave].keys():
            if subspace_master is None:
                dof_m = fem.locate_dofs_geometrical(V, master) + local_min
                dof_m = numpy.vstack(MPI.comm_world.allgather(dof_m))[0]
            else:
                dof_m = fem.locate_dofs_geometrical((V.sub(subspace_master),
                                                     Vsub_master),
                                                    master)
                for (i, d) in enumerate(dof_m):
                    dof_m[i] += local_min
                dof_m = numpy.vstack(MPI.comm_world.allgather(dof_m))[0, 0]

            masters.append(dof_m)
            coeffs.append(slave_master_dict[slave][master])
    offsets.append(len(masters))
    return (numpy.array(slaves), numpy.array(masters),
            numpy.array(coeffs, dtype=numpy.float64), numpy.array(offsets))

def dof_close_to(x, point):
    """
    Convenience function for locating a dof close to a point use numpy
    and lambda functions.
    """
    if point is None:
        raise ValueError("Point must be supplied")
    if len(point) == 1:
        return numpy.isclose(x[0], point[0])
    elif len(point) == 2:
        return numpy.logical_and(numpy.isclose(x[0], point[0]),
                                 numpy.isclose(x[1], point[1]))
    elif len(point) == 3:
        return numpy.logical_and(
            numpy.logical_and(numpy.isclose(x[0], point[0]),
                              numpy.isclose(x[1], point[1])),
            numpy.isclose(x[2], point[2]))
    else:
        return ValueError("Point has to be 1D, 2D or 3D")

def facet_normal_approximation(V, mt, mt_id):
    import dolfinx
    import ufl
    n = dolfinx.FacetNormal(V.mesh)
    nh = dolfinx.Function(V)
    u, v = ufl.TrialFunction(V), ufl.TestFunction(V)
    a = (dolfinx.Constant(V.mesh, 0)*ufl.inner(u, v)*ufl.dx
         + ufl.inner(u, v)*ufl.ds)
    ds = ufl.ds(domain=V.mesh, subdomain_data=mt, subdomain_id=mt_id)
    L = ufl.inner(n, v)*ds

    A = dolfinx.fem.assemble_matrix(a)
    ident_zeros(A)
    A.assemble()
    b = dolfinx.fem.assemble_vector(L)
    ksp = PETSc.KSP().create(V.mesh.mpi_comm())
    ksp.setOperators(A)
    ksp.setType("preonly")
    ksp.getPC().setType("lu")
    ksp.getPC().setFactorSolverType("mumps")
    ksp.solve(b, nh.vector)

    return nh

def ident_zeros(A):
    """
    Find all rows in a matrix that is zero, and add a 1 on the diagonal
    """
    assert A.size[0] == A.size[1]
    A.assemble()
    o_range = A.getOwnershipRange()
    rows = []
    for i in range(o_range[1]-o_range[0]):
        indices, values = A.getRow(o_range[0]+i)
        absrow = sum(abs(values))
        if absrow < 1e-6:
            rows.append(o_range[0] + i)
    add_diagonal(A.handle, numpy.array(rows))


'''This script loads volumes described in a "slice" of volume IDs, and
    counts words at the page level in the "clean.txt" file.

    That work gets done in the Volume module.

'''

import FileCabinet
import Volume
import Context
import sys

# DEFINE CONSTANTS.
delim = '\t'
debug = False

# LOAD PATHS.
slicename = sys.argv[1]
outfilename = sys.argv[2]

## We assume the slice name has been passed in as an argument.

pathdictionary = FileCabinet.loadpathdictionary('/home/tunder/python/tokenize/PathDictionary.txt')

datapath = pathdictionary['datapath']
slicepath = pathdictionary['slicepath'] + slicename + '.txt'
metadatapath = pathdictionary['metadatapath']
metaoutpath = pathdictionary['slicepath'] + slicename + 'acc.txt'
errorpath = pathdictionary['slicepath'] + slicename + 'errorlog.txt'
longSpath = pathdictionary['slicepath'] + slicename + 'longS.txt'

with open(slicepath, encoding="utf-8") as file:
    HTIDlist = file.readlines()

HTIDs = set()

for thisID in HTIDlist:
    thisID = thisID.rstrip()
    HTIDs.add(thisID)

del HTIDlist

## discard bad volume IDs

with open(metadatapath + "badIDs.txt", encoding = 'utf-8') as file:
    filelines = file.readlines()

for line in filelines:
    line = line.rstrip()
    line = line.split(delim)
    if line[0] in HTIDs:
        HTIDs.discard(line[0])

processedmeta = list()
errorlog = list()
longSfiles = list()
totaladded = dict()
totaldeleted = dict()

for thisID in HTIDs:

    filepath, postfix = FileCabinet.pairtreepath(thisID, datapath)
    filename = filepath + postfix + '/' + postfix + ".clean.txt"

    try:
        with open(filename, encoding='utf-8') as file:
            lines = file.readlines()
            successflag = True
    except IOError as e:
        successflag = False

    if not successflag:
        print(thisID + " is missing.")
        errorlog.append(thisID + '\t' + "missing")
        continue

    tokens, aretokensverse, pagefeatures = Volume.as_stream(lines, verbose=debug)

    tokencount = len(tokens)

    if len(tokens) < 10:
        print(thisID, "has only tokencount", len(tokens))
        errorlog.append(thisID + '\t' + 'short')

    correct_tokens, pages = Volume.correct_stream(tokens, aretokensverse, verbose = debug)
    if not correct_tokens == tokens:
        errorlog.append(thisID + '\t' + 'corrected stream changed.')

    assert len(pagefeatures) == len(pages)

with open(outfilename, mode = 'a', encoding = 'utf-8') as file:
    for index, page in enumerate(pages):
        textlines, caplines = pagefeatures[index]

        file.write(thisid + ',' + str(index) + ',' + "0" + "," + "#textlines" + ',' + str(textlines) + '\n')
        file.write(thisid + ',' + str(index) + ',' + "0" + "," + "#caplines" + ',' + str(caplines) + '\n')

        for key, value in page.items():
            word, verseflag = key
            if "," in word:
                continue
            ## Don't want to mess up my csv.

            file.write(thisid + ',' + str(index) + ',' + verseflag + "," + word + ',' + str(value) + '\n')

# Write the errorlog.

if len(errorlog) > 0:
    with open(errorpath, mode = 'w', encoding = 'utf-8') as file:
        for line in errorlog:
            file.write(line + '\n')

# Done.


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('hx_lti_initializer', '0006_lticourse_course_external_css_default'),
    ]

    operations = [
        migrations.AlterField(
            model_name='lticourse',
            name='course_external_css_default',
            field=models.CharField(blank=True, help_text=b'Please only input a URL to an externally hosted CSS file.', max_length=255, validators=[b'URLValidator']),
            preserve_default=True,
        ),
    ]


from django.conf.urls import url

from . import views

app_name = "api"

urlpatterns = [
    url(r"^plog/$", views.blogitems, name="blogitems"),
    url(r"^plog/hits/$", views.blogitem_hits, name="blogitem_hits"),
    url(
        r"^plog/realtimehits/$",
        views.blogitem_realtimehits,
        name="blogitem_realtimehits",
    ),
    url(
        r"^plog/spam/patterns$",
        views.spam_comment_patterns,
        name="spam_comment_patterns",
    ),
    url(
        r"^plog/spam/patterns/(?P<id>\d+)$",
        views.spam_comment_patterns,
        name="spam_comment_patterns",
    ),
    url(r"^plog/comments/geo/$", views.geocomments, name="geocomments"),
    url(r"^plog/comments/$", views.blogcomments, name="blogcomments"),
    url(
        r"^plog/comments/(?P<action>approve|delete)/$",
        views.blogcomments_batch,
        name="blogcomments_batch",
    ),
    url(r"^plog/preview/$", views.preview, name="preview"),
    url(r"^plog/(.*)/images$", views.images, name="images"),
    url(r"^plog/(.*)/hits$", views.hits, name="hits"),
    url(
        r"^plog/(.*)/open-graph-image$", views.open_graph_image, name="open_graph_image"
    ),
    url(r"^plog/(.*)/awspa$", views.awspa, name="awspa"),
    url(r"^plog/(.*)$", views.blogitem, name="blogitem"),
    url(r"^categories/?$", views.categories, name="categories"),
    url(r"^postprocessings/", views.postprocessings, name="postprocessings"),
    url(r"^searchresults/", views.searchresults, name="searchresults"),
    url(r"^cdn/check", views.cdn_check, name="cdn_check"),
    url(r"^cdn/config", views.cdn_config, name="cdn_config"),
    url(
        r"^cdn/purge/urls/count",
        views.cdn_purge_urls_count,
        name="cdn_purge_urls_count",
    ),
    url(r"^cdn/purge/urls", views.cdn_purge_urls, name="cdn_purge_urls"),
    url(r"^cdn/purge", views.cdn_purge, name="cdn_purge"),
    url(r"^cdn/probe", views.cdn_probe, name="cdn_probe"),
    url(
        r"lyrics-page-healthcheck",
        views.lyrics_page_healthcheck,
        name="lyrics_page_healthcheck",
    ),
    url(r"xcache/analyze", views.xcache_analyze, name="xcache_analyze"),
    url(r"", views.catch_all, name="catch_all"),
]


import os
from pathlib import Path
from unittest.mock import MagicMock, Mock

import pytest
from click import FileError, NoSuchOption

from robotidy.cli import read_config, validate_regex
from robotidy.files import find_project_root, read_pyproject_config, get_paths, DEFAULT_EXCLUDES
from robotidy.transformers import load_transformers
from robotidy.transformers.AlignSettingsSection import AlignSettingsSection
from robotidy.transformers.ReplaceRunKeywordIf import ReplaceRunKeywordIf
from robotidy.transformers.SmartSortKeywords import SmartSortKeywords
from robotidy.utils import node_within_lines
from robotidy.version import __version__
from .utils import run_tidy

@pytest.fixture
def test_data_dir():
    return Path(__file__).parent / 'testdata'

class TestCli:
    @pytest.mark.parametrize('name, similar', [
        ('NotExisting', ''),
        ('AlignSettings', ' Did you mean:\n    AlignSettingsSection'),
        ('align', ' Did you mean:\n    AlignSettingsSection\n    AlignVariablesSection'),
        ('splittoolongline', ' Did you mean:\n    SplitTooLongLine'),
        ('AssignmentNormalizer', ' Did you mean:\n    NormalizeAssignments')
    ])
    def test_not_existing_transformer(self, name, similar):
        expected_output = f"Importing transformer '{name}' failed. " \
                          f"Verify if correct name or configuration was provided.{similar}"
        args = f'--transform {name} --transform MissingTransformer --transform DiscardEmptySections -'.split()
        result = run_tidy(args, exit_code=1)
        assert expected_output in str(result.exception)

    def test_transformer_order(self):
        order_1 = ['NormalizeSeparators', 'OrderSettings']
        order_2 = ['OrderSettings', 'NormalizeSeparators']
        transformers_1 = load_transformers([(transf, []) for transf in order_1], {})
        transformers_2 = load_transformers([(transf, []) for transf in order_2], {})
        assert all(t1.__class__.__name__ == t2.__class__.__name__ for t1, t2 in zip(transformers_1, transformers_2))

    def test_transformer_force_order(self):
        # default_order = ['NormalizeSeparators', 'OrderSettings']
        custom_order = ['OrderSettings', 'NormalizeSeparators']
        transformers = load_transformers([(transf, []) for transf in custom_order], {}, force_order=True)
        assert all(t1.__class__.__name__ == t2 for t1, t2 in zip(transformers, custom_order))

    # TODO: raise exception if kwarg does not match
    # def test_not_existing_configurable(self):
    #     expected_output = "Usage: cli [OPTIONS] [PATH(S)]\n\n" \
    #                       "Error: Invalid configurable name: 'missing_configurable' for transformer: " \
    #                       "'DiscardEmptySections'\n"
    #
    #     args = '--transform DiscardEmptySections:allow_only_commentss=True -'.split()
    #     result = run_tidy(args, exit_code=2)
    #     assert expected_output == result.output

    def test_invalid_configurable_usage(self):
        expected_output = "Importing transformer 'DiscardEmptySections=allow_only_comments=False' failed. " \
                          "Verify if correct name or configuration was provided"
        args = '--transform DiscardEmptySections=allow_only_comments=False -'.split()
        result = run_tidy(args, exit_code=1)
        assert expected_output in str(result.exception)

    def test_too_many_arguments_for_transform(self):
        expected_output = "not enough values to unpack (expected 2, got 1)"
        args = '--transform DiscardEmptySections:allow_only_comments:False -'.split()
        result = run_tidy(args, exit_code=1)
        assert str(result.exception) == expected_output

    # def test_invalid_argument_type_for_transform(self):
    #     expected_output = "Importing 'robotidy.transformers.DiscardEmptySections' failed:  'DicardEmptySection'"
    #     args = '--transform DiscardEmptySections:allow_only_comments=true'.split()
    #     result = run_tidy(args, exit_code=1)
    #     assert expected_output == str(result.exception)

    def test_find_project_root_from_src(self, test_data_dir):
        src = test_data_dir / 'nested' / 'test.robot'
        path = find_project_root((src,))
        assert path == test_data_dir / 'nested'

    def test_read_robotidy_config(self, test_data_dir):
        """ robotidy.toml follows the same format as pyproject starting from 1.2.0 """
        expected_config = {
            'overwrite': False,
            'diff': False,
            'spacecount': 4,
            'transform': [
                'DiscardEmptySections:allow_only_comments=True',
                'ReplaceRunKeywordIf'
            ]
        }
        config_path = str(test_data_dir / 'config' / 'robotidy.toml')
        config = read_pyproject_config(config_path)
        assert config == expected_config

    def test_read_pyproject_config(self, test_data_dir):
        expected_parsed_config = {
            'overwrite': False,
            'diff': False,
            'startline': 10,
            'endline': 20,
            'transform': [
                'DiscardEmptySections:allow_only_comments=True',
                'SplitTooLongLine'
            ],
            'configure': [
                'DiscardEmptySections:allow_only_comments=False',
                'OrderSettings: keyword_before = documentation,tags,timeout,arguments'
            ]
        }
        config_path = str(test_data_dir / 'only_pyproject' / 'pyproject.toml')
        config = read_pyproject_config(config_path)
        assert config == expected_parsed_config

    def test_read_pyproject_config_e2e(self, test_data_dir):
        expected_parsed_config = {
            'overwrite': 'False',
            'diff': 'False',
            'startline': '10',
            'endline': '20',
            'transform': [
                'DiscardEmptySections:allow_only_comments=True',
                'SplitTooLongLine'
            ],
            'configure': [
                'DiscardEmptySections:allow_only_comments=False',
                'OrderSettings: keyword_before = documentation,tags,timeout,arguments'
            ]
        }
        config_path = str(test_data_dir / 'only_pyproject')
        ctx_mock = MagicMock()
        ctx_mock.params = {'src': (config_path,)}
        ctx_mock.command.params = None
        param_mock = Mock()
        read_config(ctx_mock, param_mock, value=None)
        assert ctx_mock.default_map == expected_parsed_config

    def test_read_invalid_config(self, test_data_dir):
        config_path = str(test_data_dir / 'invalid_pyproject' / 'pyproject.toml')
        with pytest.raises(FileError) as err:
            read_pyproject_config(config_path)
        assert 'Error reading configuration file: ' in str(err)

    @pytest.mark.parametrize('option, correct', [
        ('confgure', 'configure'),
        ('idontexist', None)
    ])
    def test_read_invalid_option_config(self, option, correct, test_data_dir):
        config_path = str(test_data_dir / 'invalid_options_config' / f'pyproject_{option}.toml')
        ctx_mock = MagicMock()
        param_mock = MagicMock()
        with pytest.raises(NoSuchOption) as err:
            read_config(ctx_mock, param_mock, config_path)
        similar = '' if correct is None else f' Did you mean {correct}'
        assert f'no such option: {option}{similar}'

    def test_read_config_from_param(self, test_data_dir):
        expected_parsed_config = {
            'overwrite': 'False',
            'diff': 'False',
            'spacecount': '4',
            'transform': [
                'DiscardEmptySections:allow_only_comments=True',
                'ReplaceRunKeywordIf'
            ]
        }
        config_path = str(test_data_dir / 'config' / 'robotidy.toml')
        ctx_mock = MagicMock()
        ctx_mock.command.params = None
        param_mock = Mock()
        read_config(ctx_mock, param_mock, config_path)
        assert ctx_mock.default_map == expected_parsed_config

    def test_read_config_without_param(self, test_data_dir):
        expected_parsed_config = {
            'overwrite': 'False',
            'diff': 'False',
            'spacecount': '4',
            'transform': [
                'DiscardEmptySections:allow_only_comments=True',
                'ReplaceRunKeywordIf'
            ]
        }
        config_path = str(test_data_dir / 'config' / 'robotidy.toml')
        ctx_mock = MagicMock()
        ctx_mock.params = {'src': (config_path,)}
        ctx_mock.command.params = None
        param_mock = Mock()
        read_config(ctx_mock, param_mock, value=None)
        assert ctx_mock.default_map == expected_parsed_config

    @pytest.mark.parametrize('node_start, node_end, start_line, end_line, expected', [
        (15, 30, 15, None, True),
        (15, 30, 15, 30, True),
        (14, 30, 15, 30, False),
        (15, 31, 15, 30, False),
        (15, 30, None, 30, True),
        (15, 30, None, None, True)
    ])
    def test_skip_node_start_end_line_setting(self, node_start, node_end, start_line, end_line, expected):
        assert node_within_lines(node_start, node_end, start_line, end_line) == expected

    @pytest.mark.parametrize('flag', ['--list', '-l'])
    def test_list_transformers(self, flag):
        result = run_tidy([flag])
        assert 'To see detailed docs run --desc <transformer_name> or --desc all. Transformers with (disabled) ' \
               'tag \nare executed only when selected explicitly with --transform or configured with param ' \
               '`enabled=True`.\n' \
               'Available transformers:\n'\
               in result.output
        assert 'ReplaceRunKeywordIf\n' in result.output
        assert 'SmartSortKeywords (disabled)\n' in result.output  # this transformer is disabled by default
        assert 'Available transformers:\n\nAddMissingEnd\n' in result.output  # assert order

    @pytest.mark.parametrize('flag', ['--desc', '-d'])
    @pytest.mark.parametrize('name, expected_doc', [
        ('ReplaceRunKeywordIf', ReplaceRunKeywordIf.__doc__.replace('::', ':').replace("``", "'")),
        ('SmartSortKeywords', SmartSortKeywords.__doc__.replace('::', ':').replace("``", "'"))
    ])
    def test_describe_transformer(self, flag, name, expected_doc):
        not_expected_doc = AlignSettingsSection.__doc__.replace('::', ':').replace("``", "'")
        result = run_tidy([flag, name])
        assert expected_doc in result.output
        assert not_expected_doc not in result.output

    def test_describe_transformer_all(self):
        expected_doc = ReplaceRunKeywordIf.__doc__.replace('::', ':').replace("``", "'")
        expected_doc2 = AlignSettingsSection.__doc__.replace('::', ':').replace("``", "'")
        result = run_tidy(['--desc', 'all'])
        assert expected_doc in result.output
        assert expected_doc2 in result.output

    @pytest.mark.parametrize('name, similar', [
        ('NotExisting', ''),
        ('AlignSettings', ' Did you mean:\n    AlignSettingsSection'),
        ('align', ' Did you mean:\n    AlignSettingsSection\n    AlignVariablesSection'),
        ('splittoolongline', ' Did you mean:\n    SplitTooLongLine'),
        ('AssignmentNormalizer', ' Did you mean:\n    NormalizeAssignments')
    ])
    def test_describe_invalid_transformer(self, name, similar):
        expected_output = f"Transformer with the name '{name}' does not exist.{similar}"
        args = f'--desc {name} -'.split()
        result = run_tidy(args, exit_code=1)
        assert expected_output in str(result.output)

    @pytest.mark.parametrize('flag', ['--help', '-h'])
    def test_help(self, flag):
        result = run_tidy([flag])
        assert f'Version: {__version__}' in result.output

    @pytest.mark.parametrize('source, return_status', [
        ('golden.robot', 0),
        ('not_golden.robot', 1)
    ])
    def test_check(self, source, return_status, test_data_dir):
        source = test_data_dir / 'check' / source
        run_tidy(
            ['--check', '--overwrite', '--transform', 'NormalizeSectionHeaderName', str(source)],
            exit_code=return_status
        )

    def test_diff(self, test_data_dir):
        source = test_data_dir / 'check' / 'not_golden.robot'
        result = run_tidy(['--diff', '--no-overwrite', '--transform', 'NormalizeSectionHeaderName', str(source)])
        assert "*** settings ***" in result.output
        assert "*** Settings ***" in result.output

    def test_disabled_transformer(self):
        transformers = load_transformers(None, {})
        assert all(transformer.__class__.__name__ != 'SmartSortKeywords' for transformer in transformers)

    def test_enable_disable_transformer(self):
        transformers = load_transformers([('SmartSortKeywords', [])], {})
        assert transformers[0].__class__.__name__ == 'SmartSortKeywords'

    def test_configure_transformer(self):
        transformers = load_transformers(
            None,
            {'AlignVariablesSection': ['up_to_column=4']}
        )
        transformers_not_configured = load_transformers(None, {})
        assert len(transformers) == len(transformers_not_configured)
        for transformer in transformers:
            if transformer.__class__.__name__ == 'AlignVariablesSection':
                assert transformer.up_to_column + 1 == 4

    def test_configure_transformer_overwrite(self):
        transformers = load_transformers(
            [('AlignVariablesSection', ['up_to_column=3'])],
            {'AlignVariablesSection': ['up_to_column=4']}
        )
        assert transformers[0].up_to_column + 1 == 4

    @pytest.mark.parametrize('line_sep', ['unix', 'windows', 'native', None])
    def test_line_sep(self, line_sep, test_data_dir):
        source = test_data_dir / 'line_sep' / 'test.robot'
        expected = test_data_dir / 'line_sep' / 'expected.robot'
        actual = test_data_dir.parent / 'actual' / 'test.robot'
        if line_sep is not None:
            run_tidy(['--lineseparator', line_sep, str(source)], output='test.robot')
        else:
            run_tidy([str(source)], output='test.robot')
        line_end = {
            'unix': '\n',
            'windows': '\r\n',
            'native': os.linesep,
            None: os.linesep
        }[line_sep]
        with open(str(expected)) as f:
            expected_str = f.read()
        expected_str = expected_str.replace('\n', line_end)
        with open(str(actual), newline='') as f:
            actual_str = f.read()
        assert actual_str == expected_str, 'Line endings does not match'

    @pytest.mark.parametrize('force_order', [True, False])
    @pytest.mark.parametrize('allow_disabled', [True, False])
    @pytest.mark.parametrize('transformers, configure, present, test_for', [
        # robotidy .
        (None, {}, True, 'AlignVariablesSection'),
        # robotidy -c AlignVariablesSection:enabled=True .
        (None, {'AlignVariablesSection': ['enabled=True']}, True, 'AlignVariablesSection'),
        # robotidy -c AlignVariablesSection:enabled=false .
        (None, {'AlignVariablesSection': ['enabled=false']}, False, 'AlignVariablesSection'),
        # robotidy -c SmartSortKeywords:enabled=True .
        (None, {'SmartSortKeywords': ['enabled=True']}, True, 'SmartSortKeywords'),  # disabled by default
        # robotidy -c SmartSortKeywords:enabled=False .
        (None, {'SmartSortKeywords': ['enabled=False']}, False, 'SmartSortKeywords'),
        # robotidy --transform SmartSortKeywords:enabled=True .
        ([('SmartSortKeywords', ['enabled=True'])], {}, True, 'SmartSortKeywords'),
        # robotidy --transform NormalizeAssignments .
        ([('NormalizeAssignments', [])], {}, False, 'AlignVariablesSection'),
        # robotidy --transform NormalizeAssignments --transform AlignVariablesSection .
        ([('NormalizeAssignments', []), ('AlignVariablesSection', [])], {}, True, 'AlignVariablesSection'),
        # robotidy --transform NormalizeAssignments --transform AlignVariablesSection:up_to_column=4 .
        ([('NormalizeAssignments', []), ('AlignVariablesSection', ['up_to_column=4'])], {}, True,
         'AlignVariablesSection'),
        # robotidy --transform NormalizeAssignments --transform AlignVariablesSection:up_to_column=4:enabled=True .
        ([('NormalizeAssignments', []), ('AlignVariablesSection', ['up_to_column=4', 'enabled=True'])], {}, True,
         'AlignVariablesSection'),
        # robotidy --transform NormalizeAssignments --transform AlignVariablesSection:up_to_column=4:enabled=False .
        ([('NormalizeAssignments', []), ('AlignVariablesSection', ['up_to_column=4', 'enabled=False'])], {}, False,
         'AlignVariablesSection'),
        # robotidy --transform NormalizeAssignments --transform AlignVariablesSection:up_to_column=4 -c
        # AlignVariablesSection:enabled=True .
        ([('NormalizeAssignments', []), ('AlignVariablesSection', ['up_to_column=4'])],
         {'AlignVariablesSection': ['enabled=True']}, True, 'AlignVariablesSection'),
        # robotidy --transform NormalizeAssignments --transform AlignVariablesSection:up_to_column=4 -c
        # AlignVariablesSection:enabled=False .
        ([('NormalizeAssignments', []), ('AlignVariablesSection', ['up_to_column=4'])],
         {'AlignVariablesSection': ['enabled=False']}, False, 'AlignVariablesSection')
    ])
    def test_disable_transformers(self, transformers, configure, present, force_order, allow_disabled, test_for):
        if force_order and not transformers:
            present = False
        loaded_transformers = load_transformers(transformers, configure, allow_disabled=allow_disabled,
                                                force_order=force_order)
        if present:
            assert any(transformer.__class__.__name__ == test_for for transformer in loaded_transformers)
        else:
            assert all(transformer.__class__.__name__ != test_for for transformer in loaded_transformers)

    @pytest.mark.parametrize('exclude, extend_exclude, allowed', [
        (DEFAULT_EXCLUDES, None, ['nested/test.robot', 'test.resource', 'test.robot']),
        ('test.resource', None, ['test.robot', 'nested/test.robot']),
        (DEFAULT_EXCLUDES, 'test.resource', ['test.robot', 'nested/test.robot']),
        ('test.resource', 'nested/*', ['test.robot'])
    ])
    def test_exclude_gitignore(self, exclude, extend_exclude, allowed, test_data_dir):
        source = test_data_dir / 'gitignore'
        allowed_paths = {Path(source, path) for path in allowed}
        paths = get_paths((str(source),), exclude=validate_regex(exclude),
                          extend_exclude=validate_regex(extend_exclude))
        assert paths == allowed_paths

    @pytest.mark.parametrize('source, should_parse', [
        (None, ['test.robot', 'resources/test.robot']),  # calls: robotidy
        ('test3.robot', ['test3.robot']),  # calls: robotidy test3.robot
        ('test.robot', ['test.robot']),
        ('.', ['test.robot', 'test3.robot', 'resources/test.robot']),
    ])
    def test_src_in_configuration(self, source, should_parse, test_data_dir):
        source_dir = test_data_dir / 'pyproject_with_src'
        os.chdir(source_dir)
        if source is not None:
            source = source_dir / source
            result = run_tidy([str(source)])
        else:
            result = run_tidy()
        expected = [f"Loaded configuration from {source_dir / 'pyproject.toml'}"]
        for file in should_parse:
            path = source_dir / file
            expected.append(f"Transforming {path} file")
        actual = sorted(line for line in result.output.split('\n') if line.strip())
        assert actual == sorted(expected)

    @pytest.mark.parametrize('source', [1, 2])
    def test_empty_configuration(self, source, test_data_dir):
        config_dir = test_data_dir / f'empty_pyproject{source}'
        os.chdir(config_dir)
        result = run_tidy(exit_code=1)
        assert "Loaded configuration from" not in result.output

    def test_loading_from_stdin(self):
        input_file = '*** Settings ***\nLibrary  SomeLib\n\n\n' \
                     '*** Variables ***\n\n\n\n' \
                     '*** Keywords ***\nKeyword\n    Keyword1 ${arg}\n'
        expected_output = '*** Settings ***\nLibrary  SomeLib\n\n\n' \
                          '*** Keywords ***\nKeyword\n    Keyword1 ${arg}\n\n'
        args = '--transform DiscardEmptySections -'.split()
        result = run_tidy(args, std_in=input_file)
        assert result.output == expected_output