from qa_engine.base import QABase
from qa_engine.score_answers import main as score_answers
import re, sys, nltk, operator
from nltk.stem.wordnet import WordNetLemmatizer
from nltk.corpus import wordnet
import csv
from word2vec import baseline_word2vec
from sklearn.metrics.pairwise import cosine_similarity
from word2vec_extractor import Word2vecExtractor
from dependency_demo_stub import find_main
from qa_engine.base import QABase
from collections import defaultdict

def get_sentences(text):
    sentences = nltk.sent_tokenize(text)
    sentences = [nltk.word_tokenize(sent) for sent in sentences]
    sentences = [nltk.pos_tag(sent) for sent in sentences]

    return sentences

DATA_DIR = "./wordnet"

QUESTIONS = ['What', 'Where', 'Who', 'When', 'Why', 'Which', 'How', 'Did', 'what', 'where', 'who', 'when', 'why', 'which', 'did', 'how']

GRAMMAR =   """
            N: {<PRP>|<NN.*>}
            V: {<V.*>}
            ADJ: {<JJ.*>}
            NP: {<DT>? <ADJ>* <N>+}
            PP: {<IN> <NP>}
            VP: {<TO>? <V> (<NP>|<PP>)*}
            """

# FIND ALL PHRASES FROM PHRASE ALL THE WAY TO NEXT PHRASE BY CHECKING KEYWORDS
# USE SUBTREES DEPENDING ON QTYPE
LOC_PP = set(["in", "on", "at"])
TIME = set(['today', 'Today', 'yesterday', 'Yesterday', 'tomorrow', 'Tomorrow', 'later', 'Later', 'Now', 'now'])
NEGATIVE = ['not', 'never', 'no']
WHERE_PATTERNS = ['(VP (PP))', '(VP (NP))']#'(PP)', '(VP (*) (PP))', '(VP (*) (NP))']
WHAT_PATTERNS = ['(VP (NP))', '(NP (VP))']#'(NP)', '(NP (*) (VP))', '(VP (*) (NP))', '(NP)']
WHO_PATTERNS = ['(NP (VP))']#, '(NP (*) (VP))', '(VP (*) (NP))']
WHY_PATTERNS = ['(SBAR)', '(PP)', '(PP (NP))']
WHEN_PATTERNS = ['(VP (NP))', '(NP (VP))']#'(NP)']
HOW_PATTERNS = ['(VP (NP))', '(NP (VP))', '(VP)', '(NP)']#'(NP (*) (VP))', '(VP (*) (NP))', '(NP)', '(VP)']
WHICH_PATTERNS = ['(VP (NP))', '(NP (VP))']

def get_answer(question, story):
    """
    :param question: dict
    :param story: dict
    :return: str

    question is a dictionary with keys:
        dep -- A list of dependency graphs for the question sentence.
        par -- A list of constituency parses for the question sentence.
        text -- The raw text of story.
        sid --  The story id.
        difficulty -- easy, medium, or hard
        type -- whether you need to use the 'sch' or 'story' versions
                of the .
        qid  --  The id of the question.

    story is a dictionary with keys:
        story_dep -- list of dependency graphs for each sentence of
                    the story version.
        sch_dep -- list of dependency graphs for each sentence of
                    the sch version.
        sch_par -- list of constituency parses for each sentence of
                    the sch version.
        story_par -- list of constituency parses for each sentence of
                    the story version.
        sch --  the raw text for the sch version.
        text -- the raw text for the story version.
        sid --  the story id

    """
    ###     Your Code Goes Here         ###

    lmtzr = WordNetLemmatizer()
    qtype, keywords = find_keywords(question, lmtzr)

    index = find_sentence(question, story, keywords)
    print(question['text'])

    answer = find_answer(qtype, question['type'], index, story, keywords)
    print(answer)

    with open('hw8-responses.csv', 'a') as csvFile:
    	writer = csv.writer(csvFile)
    	writer.writerow([answer, question['qid']])
    csvFile.close()

    ###     End of Your Code         ###
    return answer

def find_sentence(question, story, keywords):
	glove_w2v_file = "data/glove-w2v.txt"
	W2vecextractor = Word2vecExtractor(glove_w2v_file)
	stopwords = set(nltk.corpus.stopwords.words("english"))

	if question['type'] == 'Story':
		text = nltk.sent_tokenize(story['text'])
		keywords_no_tags = [word for (word, tag) in keywords]
		answer = baseline_word2vec(question['text'], text, stopwords, W2vecextractor)

	else:
		text = nltk.sent_tokenize(story['sch'])
		keywords_no_tags = [word for (word, tag) in keywords]
		answer = baseline_word2vec(question['text'], text, stopwords, W2vecextractor)
	return answer

def matches(pattern, subtree, count):
	# Found a match and tree is finished
	if subtree is None and pattern is None:
		return subtree
	# Found a match with extra nodes
	elif pattern is None:
		return subtree
	# Did not find a match
	elif subtree is None:
		return None

	plabel = pattern if isinstance(pattern, str) else pattern.label()
	tlabel = subtree if isinstance(subtree, str) else subtree.label()

	if plabel != tlabel and count != 0:
		return subtree
	elif plabel == tlabel and count == 0:
		for pchild in pattern:
			for tchild in subtree:
				match = matches(pchild, tchild, count + 1)
				if match is None:
					return None
		return subtree
	elif plabel == tlabel and count != 0:
		return subtree

	return None

def find_pattern(pattern, tree):
	for subtree in tree.subtrees():
		count = 0
		node = matches(pattern, subtree, count)
		if node is not None:
			return node
	return None

def load_wordnet_ids(filename):
	file = open(filename, 'r')
	if "noun" in filename:
		type = "noun"
	else:
		type = "verb"
	csvreader = csv.DictReader(file, delimiter=",", quotechar='"')
	word_ids = defaultdict()
	for line in csvreader:
		word_ids[line['synset_id']] = {'synset_offset': line['synset_offset'],
									   'story_' + type: line['story_' + type], 'stories': line['stories']}
	return word_ids

def find_answer(qtype, version, index, story, keywords):
	## NEW ##

	noun_ids = load_wordnet_ids("{}/{}".format(DATA_DIR, "Wordnet_nouns.csv"))
	verb_ids = load_wordnet_ids("{}/{}".format(DATA_DIR, "Wordnet_verbs.csv"))

	for synset_id, items in noun_ids.items():
		noun = items['story_noun']
		stories = items['stories']

	for synset_id, items in verb_ids.items():
		verb = items['story_verb']
		stories = items['stories']

	synset_array = []
	synset_hypo = []
	synset_hyper = []
	for k in keywords:
		synset_array.append(wordnet.synsets(k[0]))
	for synsets in synset_array:
		for synset_singular in synsets:
			synset_hypo.append(synset_singular.hyponyms())
			synset_hyper.append(synset_singular.hypernyms())
	synset_hypo = [t for t in synset_hypo if t]
	synset_hyper = [t for t in synset_hyper if t]

	synset_array_names = []
	synset_array_tags = []
	for synset in synset_array:
		for syn in synset:
			synset_array_names.append(syn.name().split('.')[0])

	synset_hypo_names = []
	for hyponym_list in synset_hypo:
		for hyponym in hyponym_list:
			synset_hypo_names.append(hyponym.name().split('.')[0])

	synset_hyper_names = []
	for hypernym_list in synset_hyper:
		for hypernym in hypernym_list:
			synset_hyper_names.append(hypernym.name().split('.')[0])
	##NEED TO FIGURE OUT HOW TO ACCESS THE STORY STRING SO WE CAN CHECK TO SEE IF THE HYPONYMS AND LEMMAS ARE ACTUALLY
	##IN THE STORY
	sentences_from_story = nltk.sent_tokenize(str(story))
	sentences_from_story = [sentence for sentence in sentences_from_story if "[" not in sentence]
	words_from_sentences = []
	for sentences in sentences_from_story:
		words_from_sentences.append(nltk.word_tokenize(sentences))
	for syn_name in synset_array_names:
		for sentences_w_words in words_from_sentences:
			if syn_name in sentences_w_words:
				synset_name_w_tag = nltk.pos_tag([syn_name])
				synset_name_w_tag = synset_name_w_tag[0]
				# synset_name_w_tag = tuple(nltk.pos_tag([syn_name]))
				if synset_name_w_tag not in keywords:
					keywords.append(synset_name_w_tag)

	for syn_hypo_name in synset_hypo_names:
		for sentences_w_words in words_from_sentences:
			if syn_hypo_name in sentences_w_words:
				synset_hypo_name_w_tag = nltk.pos_tag([syn_hypo_name])
				synset_hypo_name_w_tag = synset_hypo_name_w_tag[0]
				# synset_hypo_name_w_tag = tuple(nltk.pos_tag([syn_hypo_name]))
				if synset_hypo_name_w_tag not in keywords:
					keywords.append(synset_hypo_name_w_tag)

	for syn_hyper_name in synset_hyper_names:
		for sentences_w_words in words_from_sentences:
			if syn_hyper_name in sentences_w_words:
				synset_hyper_name_w_tag = nltk.pos_tag([syn_hyper_name])
				synset_hyper_name_w_tag = synset_hyper_name_w_tag[0]
				# synset_hypo_name_w_tag = tuple(nltk.pos_tag([syn_hypo_name]))
				if synset_hyper_name_w_tag not in keywords:
					keywords.append(synset_hyper_name_w_tag)
	## NEW ##
	lmtzr = WordNetLemmatizer()
	subj_keywords = [word for (word, tag) in keywords if tag.startswith('N')]
	verb_keywords = [word for (word, tag) in keywords if tag.startswith('V')]
	if version == 'Story':
		tree = story['story_par'][index]
	else:
		tree = story['sch_par'][index]
	answer = None
	print(tree)

	if qtype == 'Where' or qtype == 'where':
		for item in WHERE_PATTERNS:
			match = False
			for subtree in tree.subtrees():
				for verb in verb_keywords:
					compare_verb = lmtzr.lemmatize(str(subtree[0][0]), wordnet.VERB)
					if re.search(str(verb), str(compare_verb)) is not None:
						match = True
				if match is True:
					pattern = nltk.ParentedTree.fromstring(item)
					potential_tree = find_pattern(pattern, subtree)
					if potential_tree is not None:
						if 'PP' in item:
							pattern = nltk.ParentedTree.fromstring('(PP)')
							answer_tree = find_pattern(pattern, potential_tree)
							if answer_tree is not None:
								answer = ' '.join(answer_tree.leaves())
								return answer
						else:
							pattern = nltk.ParentedTree.fromstring('(NP)')
							answer_tree = find_pattern(pattern, potential_tree)
							if answer_tree is not None:
								answer = ' '.join(answer_tree.leaves())
								return answer
		# If nothing was matched, return last PP
		for subtree in tree.subtrees():
			pattern = nltk.ParentedTree.fromstring('(PP)')
			subtree = find_pattern(pattern, tree)
			if subtree is not None:
				answer = ' '.join(subtree.leaves())
		if answer is not None:
			return answer
		else:
			answer = None
	elif qtype == 'Who' or qtype == 'who':
		for item in WHO_PATTERNS:
			match = False
			for subtree in tree.subtrees():
				if match is True:
					if answer is not None:
						return answer
					else:
						break
				if match is False:
					pattern = nltk.ParentedTree.fromstring(item)
					potential_tree = find_pattern(pattern, subtree)
					if potential_tree is not None:
						answer = ' '.join(potential_tree.leaves())
				for verb in verb_keywords:
					compare_verb = lmtzr.lemmatize(str(subtree[0][0]), wordnet.VERB)
					if re.search(str(verb), str(compare_verb)) is not None:
						match = True
		# If nothing was matched, return first NP
		pattern = nltk.ParentedTree.fromstring('(NP)')
		subtree = find_pattern(pattern, tree)
		if subtree is not None:
			answer = ' '.join(subtree.leaves())
			return answer
		else:
			answer = None
	elif qtype == 'What' or qtype == 'what':
		for item in WHAT_PATTERNS:
			match = False
			for subtree in tree.subtrees():
				for noun in subj_keywords:
					compare_noun = lmtzr.lemmatize(str(subtree[0][0]), wordnet.NOUN)
					if re.search(str(noun), str(compare_noun)) is not None:
						match = True
				if match is True:
					pattern = nltk.ParentedTree.fromstring(item)
					potential_tree = find_pattern(pattern, subtree)
					if potential_tree is not None:
						pattern = nltk.ParentedTree.fromstring('(NP)')
						answer_tree = find_pattern(pattern, potential_tree)
						if answer_tree is not None:
							answer = ' '.join(answer_tree.leaves())
							return answer
		# If nothing was matched, return first NP
		pattern = nltk.ParentedTree.fromstring('(NP)')
		subtree = find_pattern(pattern, tree)
		if subtree is not None:
			answer = ' '.join(subtree.leaves())
			return answer
		else:
			answer = None
	elif qtype == 'When' or qtype == 'when':
		for item in WHEN_PATTERNS:
			match = False
			for subtree in tree.subtrees():
				for verb in verb_keywords:
					compare_verb = lmtzr.lemmatize(str(subtree[0][0]), wordnet.VERB)
					if re.search(str(verb), str(compare_verb)) is not None:
						match = True
				if match == False:
					pattern = nltk.ParentedTree.fromstring(item)
					potential_tree = find_pattern(pattern, subtree)
					if potential_tree is not None:
						pattern = nltk.ParentedTree.fromstring('(NP)')
						answer_tree = find_pattern(pattern, potential_tree)
						if answer_tree is not None:
							answer = ' '.join(answer_tree.leaves())
				if match == True:
					pattern = nltk.ParentedTree.fromstring(item)
					potential_tree = find_pattern(pattern, subtree)
					if potential_tree is not None:
						pattern = nltk.ParentedTree.fromstring('(NP)')
						answer_tree = find_pattern(pattern, potential_tree)
						if answer_tree is not None:
							answer = ' '.join(answer_tree.leaves())
							return answer
		# If nothing was matched, return last NP
		for subtree in tree.subtrees():
			pattern = nltk.ParentedTree.fromstring('(NP)')
			subtree = find_pattern(pattern, tree)
			if subtree is not None:
				answer = ' '.join(subtree.leaves())
		if answer is not None:
			return answer
		else:
			answer = None
	elif qtype == 'Why' or qtype == 'why':
		for item in WHY_PATTERNS:
			match = False
			for subtree in tree.subtrees():
				for noun in subj_keywords:
					compare_noun = lmtzr.lemmatize(str(subtree[0][0]), wordnet.NOUN)
					if re.search(str(noun), str(compare_noun)) is not None:
						match = True
				if match is True:
					pattern = nltk.ParentedTree.fromstring(item)
					potential_tree = find_pattern(pattern, subtree)
					if potential_tree is not None:
						if 'PP' in item:
							pattern = nltk.ParentedTree.fromstring('(PP)')
							answer_tree = find_pattern(pattern, potential_tree)
							if answer_tree is not None:
								answer = ' '.join(answer_tree.leaves())
								return answer
						else:
							answer = ' '.join(potential_tree.leaves())
							return answer
		# If nothing was matched, return last PP
		for subtree in tree.subtrees():
			pattern = nltk.ParentedTree.fromstring('(PP)')
			subtree = find_pattern(pattern, tree)
			if subtree is not None:
				answer = ' '.join(subtree.leaves())
		if answer is not None:
			return answer
		else:
			answer = None
	elif qtype == 'Did' or qtype == 'did' or qtype == 'Had' or qtype == 'had':
		match = False
		# Checks to see if verb is in sentence
		for subtree in tree.subtrees():
			if str(subtree[0][0]) in NEGATIVE:
				return 'no'
			for verb in verb_keywords:
				compare_verb = lmtzr.lemmatize(str(subtree[0][0]), wordnet.VERB)
				if re.search(str(verb), str(compare_verb)) is not None:
					match = True
		# If verb occurred, then return 'yes', else 'no'
		if match == True:
			return 'yes'
		else:
			return 'no'
	elif qtype == 'How' or qtype == 'how':
		for item in HOW_PATTERNS:
			match = False
			for subtree in tree.subtrees():
				for noun in subj_keywords:
					compare_noun = lmtzr.lemmatize(str(subtree[0][0]), wordnet.NOUN)
					if re.search(str(noun), str(compare_noun)) is not None:
						match = True
				if match == True and subtree[0][0] not in subj_keywords:
					pattern = nltk.ParentedTree.fromstring(item)
					potential_tree = find_pattern(pattern, subtree)
					if potential_tree is not None:
						answer = ' '.join(potential_tree.leaves())
						return answer
		pattern = nltk.ParentedTree.fromstring('(NP)')
		subtree = find_pattern(pattern, tree)
		if subtree is not None:
			answer = ' '.join(subtree.leaves())
			return answer
		else:
			answer = None
	elif qtype == 'Which' or qtype == 'which':
		for item in WHICH_PATTERNS:
			match = False
			for subtree in tree.subtrees():
				for noun in subj_keywords:
					compare_noun = lmtzr.lemmatize(str(subtree[0][0]), wordnet.NOUN)
					if re.search(str(noun), str(compare_noun)) is not None:
						match = True
				if match == False:
					pattern = nltk.ParentedTree.fromstring(item)
					potential_tree = find_pattern(pattern, subtree)
					if potential_tree is not None:
						pattern = nltk.ParentedTree.fromstring('(NP)')
						answer_tree = find_pattern(pattern, potential_tree)
						if answer_tree is not None:
							answer = ' '.join(answer_tree.leaves())
				if match == True and subtree[0][0] not in subj_keywords:
					pattern = nltk.ParentedTree.fromstring(item)
					potential_tree = find_pattern(pattern, subtree)
					if potential_tree is not None:
						pattern = nltk.ParentedTree.fromstring('(NP)')
						answer_tree = find_pattern(pattern, potential_tree)
						if answer_tree is not None:
							answer = ' '.join(answer_tree.leaves())
							return answer
		pattern = nltk.ParentedTree.fromstring('(NP)')
		subtree = find_pattern(pattern, tree)
		if subtree is not None:
			answer = ' '.join(subtree.leaves())
			return answer
		else:
			answer = None

	if answer is None:
		pattern = nltk.ParentedTree.fromstring('(NP)')
		subtree = find_pattern(pattern, tree)
		if subtree is not None:
			answer = ' '.join(subtree.leaves())
			return answer
		else:
			answer = 'None'

	return answer

########## DON'T EDIT ############
def find_keywords(question, lmtzr):
    stopwords = set(nltk.corpus.stopwords.words("english"))
    question = question['text']
    if question.split()[0] in QUESTIONS:
    	qtype = question.split()[0]
    else:
    	for word in question.split():
    		if word in QUESTIONS:
    			qtype = word
    			break
    		else:
    			qtype = None
    raw_qwords = get_sentences(question)
    qwords = [word for word in raw_qwords[0] if re.search(r'\w', word[0].lower()) and word[0].lower() not in stopwords]
    keywords = []
    for (word, tag) in qwords:
    	if tag.startswith('V'):
    		keywords.append((lmtzr.lemmatize(word, wordnet.VERB), tag))
    	elif tag.startswith('N'):
    		keywords.append((lmtzr.lemmatize(word, wordnet.NOUN), tag))
    	elif tag.startswith('J'):
    		keywords.append((lmtzr.lemmatize(word, wordnet.ADJ), tag))
    	elif tag.startswith('R'):
    		keywords.append((lmtzr.lemmatize(word, wordnet.ADV), tag))

    return qtype, keywords

#############################################################
###     Dont change the code in this section
#############################################################
class QAEngine(QABase):
    @staticmethod
    def answer_question(question, story):
        answer = get_answer(question, story)
        return answer

def run_qa(evaluate=False):
    QA = QAEngine(evaluate=evaluate)
    QA.run()
    QA.save_answers()

#############################################################

def main():
    # set evaluate to True/False depending on whether or
    # not you want to run your system on the evaluation
    # data. Evaluation data predictions will be saved
    # to hw6-eval-responses.tsv in the working directory.
    with open('hw8-responses.csv', 'w') as csvFile:
    	writer = csv.writer(csvFile)
    	writer.writerow(['Answer', 'qid'])
    csvFile.close()
    run_qa(evaluate=False)
    # You can uncomment this next line to evaluate your
    # answers, or you can run score_answers.py
    score_answers()

if __name__ == "__main__":
    main()

Natural Language Processing (Python)/Question Answering System/qa.py


#-------------------------------------------------------------------------------
#
# Project: EOxServer <http://eoxserver.org>
# Authors: Stephan Krause <stephan.krause@eox.at>
#          Stephan Meissl <stephan.meissl@eox.at>
#          Fabian Schindler <fabian.schindler@eox.at>
#
#-------------------------------------------------------------------------------
# Copyright (C) 2011 EOX IT Services GmbH
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies of this Software or works derived from this Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#-------------------------------------------------------------------------------

import logging
from datetime import timedelta
from itertools import chain

from django.shortcuts import render_to_response
from django.template import RequestContext
from django.utils.timezone import now

from eoxserver.core.util.timetools import isoformat
from eoxserver.resources.coverages import models

logger = logging.getLogger(__name__)

def index(request):
    return render_to_response(
        'webclient/index.html', {
            "path": request.path,
        },
        context_instance=RequestContext(request)
    )

def configuration(request):
    collections = models.Collection.objects.all()
    coverages = filter(
        lambda c: not models.iscollection(c),
        models.Coverage.objects.filter(
            visible=True, collections__isnull=True
        )
    )

    all_objects = list(chain(collections, coverages))
    try:
        start_time = min(o.begin_time for o in all_objects if o.begin_time)
        end_time = max(o.end_time for o in all_objects if o.end_time)
    except ValueError:
        start_time = now() - timedelta(days=5)
        end_time = now()

    return render_to_response(
        'webclient/config.json', {
            "layers": all_objects,
            "start_time_full": isoformat(start_time - timedelta(days=5)),
            "end_time_full": isoformat(end_time + timedelta(days=5)),
            "start_time": isoformat(start_time),
            "end_time": isoformat(end_time)
        },
        context_instance=RequestContext(request)
    )


import db_manager
import Pyro4
Pyro4.config.SERIALIZER = 'pickle'
Pyro4.config.SERIALIZERS_ACCEPTED = {'pickle', 'json' , 'marshal', 'serpent'}

@Pyro4.expose
class DbServer:
    def __init__(self):
        self.manager = db_manager.DbManager()
        self._current_item = None
        self.incoming_buffer = []

    def register_packet(self, data):
        self.incoming_buffer.append(data)

    def search_incoming_buffer(self, pattern):
        def check_contains(attr, key):
            if key in pattern.keys() and pattern[key] != attr:
                return False
            return True

        found = [x for x in self.incoming_buffer if
                 (check_contains(x.device, 'device')
                  and check_contains(x.description, 'description')
                  and check_contains(x.date, 'date'))]
        try:
            self.current_item = found[0]
        except IndexError:
            self.current_item = None
        return found

    def save_to_db(self):
        self.manager.save_packet(self.current_item)

    def load_from_db(self):
        return self.manager.load_packet()

    def search_database(self, pattern):
        return self.manager.search(pattern)

    @property
    def db_current_item(self):
        return self.manager.item

    @property
    def current_item(self):
        return self._current_item

    @current_item.setter
    def current_item(self, current_item):
        self._current_item = current_item

    def manager_name(self):
        return self.manager.__str__()

def __simple_test():
    import timehistory
    server = DbServer()
    packet = timehistory.TimeHistory(device="aa", description="desc1", date=2018,
                                     channel_nr=1, unit='unit', resolution=0, buffer='buff', sensitivity=0)
    server.register_packet(packet)
    print(server.search_incoming_buffer({'device': 'aa'}))
    print(server.current_item)
    server.save_to_db()
    print(server.search_database({'device': 'aa'}))
    print(server.db_current_item)
    packet2 = server.load_from_db()
    print(packet2.device)

if __name__ == '__main__':
    server = DbServer()
    with Pyro4.Daemon() as daemon:
        server_uri = daemon.register(server)
        with Pyro4.locateNS() as ns:
            ns.register("remote_database_access.server", server_uri)
        print("Remote database access available.")
        daemon.requestLoop()


# -*- coding: utf-8 -*-
import unittest
import os
import autoaim
from autoaim import *

data1 = [0xA5, 0x03, 0x00, 0x3E, 0x86, 0x01,
         0x00, 0x01, 0xBC, 0x02, 0x0F, 0x74]

isTravis = bool(os.environ.get('TRAVIS_PYTHON_VERSION'))

class AutoAimTestSuite(unittest.TestCase):

    def test_crc8_checksum(self):
        crc = telegram.get_crc8_checksum(data1[0:4])
        assert crc == data1[4]

    def test_crc16_checksum(self):
        crc = telegram.get_crc16_checksum(data1[0:10])
        assert crc == (data1[10]+(data1[11] << 8))

    def test_telegram_pack(self):
        test_packet = telegram.pack(
            0x0001, [bytes([0x01, 0xBC, 0x02])], seq=0x3E)
        assert bytes(data1) == test_packet

    def test_telegram_unpack(self):
        unpacker = telegram.Unpacker()
        info = {}
        for byte in bytes(data1):
            info = unpacker.send(byte)
        assert info['state'] == 'EOF'

    @unittest.skipUnless(telegram.port_list and (not isTravis), 'No serial port available.')
    def test_telegram_send(self):
        print('--- telegram ---')
        x = telegram.send(bytearray(data1))
        print(x)
        assert x

    def test_feature(self):
        '''calculate features using test8'''
        for i in range(0, 5, 1):
            img_url = 'data/test8/img{}.jpg'.format(i)
            img = helpers.load(img_url)
            feature = Feature(img)
            feature.calc([
                'contours',
                'bounding_rects',
                'rotated_rects',
                'greyscales',
                'point_areas',
            ])

    def test_predictor(self):
        '''predicte using test8'''
        for i in range(0, 5, 1):
            img_url = 'data/test8/img{}.jpg'.format(i)
            img = helpers.load(img_url)
            predictor = Predictor(
                'weight9.csv', 'pair_weight.csv', 'angle_weight.csv')
            feature = predictor.predict(img, mode='red', debug=False)
            lamps = feature.lamps
            lamps = [x for x in lamps if x.y > 0.5]

    @helpers.time_this
    def test_fps(self):
        '''predict 100 image from test9'''
        for i in range(0, 100, 1):
            img_url = 'data/test9/img{}.jpg'.format(i)
            img = helpers.load(img_url)
            # predictor = Predictor('weight9.csv')
            # lamps = predictor.predict(img, mode='red', debug=False)
            # lamps = [x for x in lamps if x.y > 0.5]

if __name__ == '__main__':
    unittest.main()


from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
from pycorenlp import StanfordCoreNLP
import numpy as np
import pandas as pd
import pickle

class Preprocessor:
    def __init__(self):
        with open('./stopwords.txt', 'r') as f:
            stopwords = f.read()
        self.stopwords = stopwords.split('\n')

    def process(self, sentence):
        sentence = self._to_lowercase(sentence)
        sentence = self._remove_stop_words(sentence)
        sentence = self._remove_extraneous(sentence)
        return sentence

    def _to_lowercase(self, sentence):
        return sentence.lower()

    def _remove_stop_words(self, sentence):
        new = ''
        for word in sentence.split(' '):
            if word not in self.stopwords:
                new += f' {word} '
        return new

    def _remove_extraneous(self, sentence):
        new = ''
        for word in sentence.split(' '):
            if not ('http' in word or '@' in word or 'rt' in word or '…' in word):
                new += f' {word} '
        return new

class VaderSentimentAttacher:
    def __init__(self):
        self.preprocessor = Preprocessor()

    def get_vader_analyzer(self):
        return SentimentIntensityAnalyzer()

    def get_vader_sentiment(self, analyzer, sentence, compound=True):
        if compound:
            return analyzer.polarity_scores(sentence)['compound']
        return analyzer.polarity_scores(sentence)

    def attach_sentiment(self, df):
        vader_analyzer = self.get_vader_analyzer()
        df['processed_text'] = df['text'].apply(lambda x: self.preprocessor.process(x))
        df['vader_sent'] = df['processed_text'].apply(lambda x: self.get_vader_sentiment(vader_analyzer, x))
        return df

class NTA:
    def __init__(self, nta):
        self.nta = nta
        self.vader = VaderSentimentAttacher()

    def get_nta_sentiment_df(self, vader=True, stanford=False):
        with open(f'./cache/{self.nta}.pkl', 'rb') as f:
            tweets = pickle.load(f)
        dates, tweet_ids, texts = [], [], []
        for tweet in tweets:
            dates.append(tweet._json['created_at'])
            tweet_ids.append(tweet._json['id'])
            texts.append(tweet._json['text'])
        df = pd.DataFrame({'date': dates, 'tweet_id': tweet_ids, 'text': texts})
        df['nta'] = self.nta
        if vader:
            df = self.vader.attach_sentiment(df)
        return df


"""
User authentication and authorization for CherryPy

This code is based on the examples and discussion at
http://tools.cherrypy.org/wiki/AuthenticationAndAccessRestrictions

"""

__author__ = 'Michael Stella <pycrust@thismetalsky.org>'

import cherrypy
from six.moves import urllib

from pycrust import url

def check_auth(*args, **kwargs):
    """Check authentication before each handler"""

    conditions = cherrypy.request.config.get('auth.require', None)
    if conditions:
        uid = cherrypy.session.get('uid')
        if uid:
            cherrypy.request.login = uid
            for c in conditions:
                if not c():
                    raise cherrypy.HTTPError(403)

        else:
            # throw a 401 rather than redirect if this is a JSON-wanting request
            # not sure why some things set HTTP_ACCEPT and others ACCEPT
            for h in ('ACCEPT', 'HTTP_ACCEPT'):
                if (h in cherrypy.request.headers and
                    'application/json' in cherrypy.request.headers[h]):
                    raise cherrypy.HTTPError(401)

            # otherwise, for a normal request,
            # redirect to the login page, preserving the 'return' URL
            target = url('login')
            referer = cherrypy.request.request_line.split()[1]
            if referer:
                target = "%s?ret=%s" % (target,  urllib.parse.quote(referer))
            raise cherrypy.HTTPRedirect(target)

cherrypy.tools.auth = cherrypy.Tool('before_handler', check_auth)

## auth decorators
def require(*conditions):
    def decorate(f):
        if not hasattr(f, '_cp_config'):
            f._cp_config = dict()
        if 'auth.require' not in f._cp_config:
            f._cp_config['auth.require'] = []
        f._cp_config['auth.require'].extend(conditions)
        return f
    return decorate

def all_of(*conditions):
    def check():
        for c in conditions:
            if not c():
                return False
        return True
    return check

def one_of(*conditions):
    def check():
        for c in conditions:
            if c():
                return True
        return False
    return check

## auth conditions
def auth_valid_user():
    """True if there is a user logged in"""
    def check():
        if cherrypy.request.login:
            return True
        return False
    return check

def auth_user(id):
    """True if the userid matches"""
    return lambda: cherrypy.request.login == id


#!flask/bin/python

__author__ = "Abhimanyu Nagurkar"
__copyright__ = "Copyright (C) Nginx Inc. All rights reserved."
__license__ = ""
__maintainer__ = "Abhimanyu Nagurkar"
__email__ = "abhimanyu.nagurkar@nginx.com"

"""Calculates median of any given list"""
def median(lst):
    n = len(lst)
    if n < 1:
            return None
    if n % 2 == 1:
            return sorted(lst)[n//2]
    else:
            return sum(sorted(lst)[n//2-1:n//2+1])/2.0


from flask import Flask, render_template, request, redirect, Blueprint, url_for
from models.animal import Animal
from models.vet import Vet
from models.owner import Owner
import repositories.vet_repository as vet_repository
import repositories.owner_repository as owner_repository
import repositories.animal_repository as animal_repository

owners_blueprint = Blueprint("owners", __name__)

@owners_blueprint.route("/owners")
def index():
    owner_repository.create_no_owner()
    message = request.args.get('message')
    show_all = request.args.get('show_all')
    if show_all == "True":
        all_owners = owner_repository.select_all()
    else:
        all_owners = owner_repository.select_all_active()
    return render_template("owners/index.html.j2", **locals())

@owners_blueprint.route("/owners/new", methods=["POST", "GET"])
def new():
    if request.method == 'GET':
        return render_template("owners/new.html.j2")
    if request.method == 'POST':
        name = request.form['name']
        address = request.form['address']
        deactivated = False
        if name != "":
            owner = Owner(name, address, deactivated)
            saved_owner = owner_repository.save(owner)
            if saved_owner.id != None:
                message = f"Success {saved_owner.name} added"
            else:
                message = "Failure"
            if request.form['action'] == "finish":
                return redirect(url_for("owners.index", message=message))
            elif request.form['action'] == "continue":
                return render_template("owners/new.html.j2", message=message)
        else:
            return redirect(url_for("owner.index"))
    else:
        print("POST Error 405 Method Not Allowed")

@owners_blueprint.route("/owners/detail/<action>/<id>", methods=["POST", "GET"])
def detail(action, id):
    if request.method == 'GET':
        animals = animal_repository.select_all_by_owner_id(id)
        owner = owner_repository.select_id(id)
        len_animals = len(animal_repository.select_all_by_owner_id(id))
        if action == "show":
            return render_template("owners/show.html.j2", owner=owner, len_animals=len_animals)
        elif action == "edit":
            return render_template("owners/edit.html.j2", owner=owner)
        elif action == "animals":
            return render_template("owners/detail.html.j2", owner=owner, len_animals=len_animals, animals=animals)
        elif action == "orphaned":
            animals = animal_repository.select_all_by_owner_id(1)
            active_owners = owner_repository.select_all_active()
            return render_template("owners/orphaned.html.j2", active_owners=active_owners, owner=owner, len_animals=len_animals, animals=animals)
        else:
            message = "Malformed URL"
            return redirect(url_for("owners.index", message=message))
    if request.method == 'POST':
        if action == "delete":
            owner_repository.delete_id(request.form['id'])
            message = f"Owner: {owner.name} (id:{owner.id}) deleted"
            return redirect(url_for("owners.index", message=message))
        elif action == "edit":
            name = request.form['name']
            address = request.form['address']
            try:
                deactivated = request.form['deactivated']
            except:
                deactivated = False
            id = request.form['id']
            owner = Owner(name, address, deactivated, id)
            owner_repository.update(owner)
            message = f"Owner: {owner.name} (id:{owner.id}) updated"
            return redirect(url_for("owners.index", message=message))
        elif action == "deactivate":
            owner = owner_repository.select_id(id)
            animals = animal_repository.select_all_by_owner_id(id)
            for animal in animals:
                animal.owner = owner_repository.select_id(1)
                animal_repository.update(animal)
            owner.deactivated = True
            owner_repository.update(owner)
            message = f"All animals moved to No Owner, Owner {owner.name} deactivated, please reassign"
            return redirect(url_for("owners.index", message=message))
        else:
            message = "Malformed URL"
            return redirect(url_for("owners.index", message=message))


import os
import shutil

import click
import pandas as pd
from deepsense import neptune
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import train_test_split

import pipeline_config as cfg
from pipelines import PIPELINES
from utils import create_submission, init_logger, read_params, persist_evaluation_predictions, \
    set_seed, verify_submission

set_seed()
logger = init_logger()
ctx = neptune.Context()
params = read_params(ctx)

@click.group()
def action():
    pass

@action.command()
@click.option('-p', '--pipeline_name', help='pipeline to be trained', required=True)
@click.option('-d', '--dev_mode', help='if true only a small sample of data will be used', is_flag=True, required=False)
def train(pipeline_name, dev_mode):
    _train(pipeline_name, dev_mode)

@action.command()
@click.option('-p', '--pipeline_name', help='pipeline to be trained', required=True)
@click.option('-d', '--dev_mode', help='if true only a small sample of data will be used', is_flag=True, required=False)
def evaluate(pipeline_name, dev_mode):
    _evaluate(pipeline_name, dev_mode)

@action.command()
@click.option('-p', '--pipeline_name', help='pipeline to be trained', required=True)
@click.option('-d', '--dev_mode', help='if true only a small sample of data will be used', is_flag=True, required=False)
def predict(pipeline_name, dev_mode):
    _predict(pipeline_name, dev_mode)

@action.command()
@click.option('-p', '--pipeline_name', help='pipeline to be trained', required=True)
@click.option('-d', '--dev_mode', help='if true only a small sample of data will be used', is_flag=True, required=False)
def train_evaluate_predict(pipeline_name, dev_mode):
    _train(pipeline_name, dev_mode)
    _evaluate(pipeline_name, dev_mode)
    _predict(pipeline_name, dev_mode)

@action.command()
@click.option('-p', '--pipeline_name', help='pipeline to be trained', required=True)
@click.option('-d', '--dev_mode', help='if true only a small sample of data will be used', is_flag=True, required=False)
def evaluate_predict(pipeline_name, dev_mode):
    _evaluate(pipeline_name, dev_mode)
    _predict(pipeline_name, dev_mode)

@action.command()
@click.option('-p', '--pipeline_name', help='pipeline to be trained', required=True)
@click.option('-d', '--dev_mode', help='if true only a small sample of data will be used', is_flag=True, required=False)
def train_evaluate(pipeline_name, dev_mode):
    _train(pipeline_name, dev_mode)
    _evaluate(pipeline_name, dev_mode)

def _train(pipeline_name, dev_mode):
    logger.info('TRAINING')
    if bool(params.clean_experiment_directory_before_training) and os.path.isdir(params.experiment_directory):
        logger.info('Cleaning experiment_directory...')
        shutil.rmtree(params.experiment_directory)

    logger.info('Reading data...')
    if dev_mode:
        logger.info('running in "dev-mode". Sample size is: {}'.format(cfg.DEV_SAMPLE_SIZE))
        application_train = pd.read_csv(params.train_filepath, nrows=cfg.DEV_SAMPLE_SIZE)
    else:
        application_train = pd.read_csv(params.train_filepath)

    logger.info('Shuffling and splitting into train and test...')
    train_data_split, valid_data_split = train_test_split(application_train,
                                                          test_size=params.validation_size,
                                                          random_state=cfg.RANDOM_SEED,
                                                          shuffle=params.shuffle)

    logger.info('Target mean in train: {}'.format(train_data_split[cfg.TARGET_COLUMN].mean()))
    logger.info('Target mean in valid: {}'.format(valid_data_split[cfg.TARGET_COLUMN].mean()))
    logger.info('Train shape: {}'.format(train_data_split.shape))
    logger.info('Valid shape: {}'.format(valid_data_split.shape))

    data = {'input': {'X': train_data_split.drop(cfg.TARGET_COLUMN, axis=1),
                      'y': train_data_split[cfg.TARGET_COLUMN],
                      'X_valid': valid_data_split.drop(cfg.TARGET_COLUMN, axis=1),
                      'y_valid': valid_data_split[cfg.TARGET_COLUMN],
                      },
            }

    pipeline = PIPELINES[pipeline_name]['train'](cfg.SOLUTION_CONFIG)
    pipeline.clean_cache()
    logger.info('Start pipeline fit and transform')
    pipeline.fit_transform(data)
    pipeline.clean_cache()

def _evaluate(pipeline_name, dev_mode):
    logger.info('EVALUATION')
    logger.info('reading data...')
    if dev_mode:
        logger.info('running in "dev-mode". Sample size is: {}'.format(cfg.DEV_SAMPLE_SIZE))
        application_train = pd.read_csv(params.train_filepath, nrows=cfg.DEV_SAMPLE_SIZE)
    else:
        application_train = pd.read_csv(params.train_filepath)

    logger.info('Shuffling and splitting to get validation split...')
    _, valid_data_split = train_test_split(application_train,
                                           test_size=params.validation_size,
                                           random_state=cfg.RANDOM_SEED,
                                           shuffle=params.shuffle)

    logger.info('Target mean in valid: {}'.format(valid_data_split[cfg.TARGET_COLUMN].mean()))
    logger.info('Valid shape: {}'.format(valid_data_split.shape))

    y_true = valid_data_split[cfg.TARGET_COLUMN].values
    data = {'input': {'X': valid_data_split.drop(cfg.TARGET_COLUMN, axis=1),
                      'y': valid_data_split[cfg.TARGET_COLUMN],
                      },
            }

    pipeline = PIPELINES[pipeline_name]['inference'](cfg.SOLUTION_CONFIG)
    pipeline.clean_cache()
    logger.info('Start pipeline transform')
    output = pipeline.transform(data)
    pipeline.clean_cache()

    y_pred = output['clipped_prediction']

    logger.info('Saving evaluation predictions to the {}'.format(params.experiment_directory))
    persist_evaluation_predictions(params.experiment_directory,
                                   y_pred,
                                   valid_data_split,
                                   cfg.ID_COLUMN,
                                   cfg.TARGET_COLUMN)

    logger.info('Calculating ROC_AUC on validation set')
    score = roc_auc_score(y_true, y_pred)
    logger.info('ROC_AUC score on validation is {}'.format(score))
    ctx.channel_send('ROC_AUC', 0, score)

def _predict(pipeline_name, dev_mode):
    logger.info('PREDICTION')
    logger.info('reading data...')
    if dev_mode:
        logger.info('running in "dev-mode". Sample size is: {}'.format(cfg.DEV_SAMPLE_SIZE))
        application_test = pd.read_csv(params.test_filepath, nrows=cfg.DEV_SAMPLE_SIZE)
    else:
        application_test = pd.read_csv(params.test_filepath)

    data = {'input': {'X': application_test,
                      'y': None,
                      },
            }

    pipeline = PIPELINES[pipeline_name]['inference'](cfg.SOLUTION_CONFIG)
    pipeline.clean_cache()
    logger.info('Start pipeline transform')
    output = pipeline.transform(data)
    pipeline.clean_cache()
    y_pred = output['clipped_prediction']

    if not dev_mode:
        logger.info('creating submission file...')
        submission = create_submission(application_test, y_pred)

        logger.info('verifying submission...')
        sample_submission = pd.read_csv(params.sample_submission_filepath)
        verify_submission(submission, sample_submission)

        submission_filepath = os.path.join(params.experiment_directory, 'submission.csv')
        submission.to_csv(submission_filepath, index=None, encoding='utf-8')
        logger.info('submission persisted to {}'.format(submission_filepath))
        logger.info('submission head \n\n{}'.format(submission.head()))

        if params.kaggle_api:
            logger.info('making Kaggle submit...')
            os.system('kaggle competitions submit -c home-credit-default-risk -f {} -m {}'
                      .format(submission_filepath, params.kaggle_message))

if __name__ == "__main__":
    action()


'''***************************************************************************
*
* FILE NAME:
*    bta_ext_if_emif_commlib.py
*
* DESCRIPTION:
*    Read, write, open, connection for EM interface will be performed using this module
*
* REVISION HISTORY
*
* Date              Author              Reason
* 25 Nov 2009       Manish Gupta        First Draft
*
*
* Copyright 2009, ARICENT
*
***************************************************************************'''

# Import system modules
import time
import string

# Import user defined modules
import bta_util_constants as CONSTANTS
from bta_commlib_extif_lib import clsCommLibInterface

# Constants Definition
MODULE_NAME             =   "bta_ext_if_emif_commlib"

class clsEMIFCommLibInterface:
    '''
    Communicates with Communication Library for sending command to TestComplete for communicating with EM
    '''
    def __init__( self, instrument_config, logger, err_handler ):
        '''
        * FUNCTION NAME:
        *    __init__()
        *
        * DESCRIPTION:
        *    Constructor Function of clsEMIFCommLibInterface class.
        *
        *
        * INPUT:
        *    instrument_config  -   Object for reading setup configuration file (instrument details), used to read the details of EM from setup configuration
        *    logger             -   Logger class object, to log event in execution logs
        *    err_handler        -   Object of err handler thread
        *
        * RETURNS:
        *    Object of clsEMIFCommLibInterface Class
        *
        * NOTES:
        *
        '''

        self.instrument_config  = instrument_config
        self.logger             = logger
        self.err_handler        = err_handler
        self.obj_comm_lib       = clsCommLibInterface(self.instrument_config, self.logger, self.err_handler)

    def openConnection( self, tool_id ):
        '''
        * FUNCTION NAME:
        *    openConnection()
        *
        * DESCRIPTION:
        *    Creates connection handler for EM specified in setup configuration file in EM interface tag
        *
        * INPUT:
        *    tool_id   -   tool_id of device
        *
        * RETURNS:
        *    connection_handler of Element Manager
        *
        * NOTES:
        *
        '''

        try:
            return self.obj_comm_lib.openConnection(tool_id)
        except Exception as detail:
            self.logger.logString( MODULE_NAME + ".openConnection : " + str(detail), CONSTANTS.ERROR_LEVEL )
            #self.err_handler.handleError(MODULE_NAME, "Error occured in open connection : " + str(tool_id),str(detail))

    def writeCommand( self, tool_id, conn_handler, cmd_str ):
        '''
        * FUNCTION NAME:
        *    writeCommand()
        *
        * DESCRIPTION:
        *    Sends command to TestComplete for performing EM operations
        *
        *
        * INPUT:
        *    tool_id        -   tool_id of device
        *    conn_handler   -   connection handler for TestComplete
        *    cmd_str        -   command string which need to be send to TestComplete
        *
        * RETURNS:
        *    None
        *
        * NOTES:
        *
        '''

        try:
            result = CONSTANTS.FAIL
            result, response = self.obj_comm_lib.writeCommand(tool_id, conn_handler, cmd_str)
        except Exception as detail:
            result = CONSTANTS.FAIL
            self.logger.logString( MODULE_NAME + ".writeCommand : " + str(detail), CONSTANTS.ERROR_LEVEL )
            #self.err_handler.handleError(MODULE_NAME, "Error occured in sending command : " + str(tool_id),str(detail))
        finally:
            return result,response

    def readResponse( self, tool_id, conn_handler, cmd_str=None ):
        '''
        * FUNCTION NAME:
        *    readResponse()
        *
        * DESCRIPTION:
        *    Reads response from TestComplete
        *
        *
        * INPUT:
        *    tool_id        -   tool_id of device
        *    conn_handler   -   connection handler for TestComplete
        *
        * RETURNS:
        *    response of TestComplete
        *
        * NOTES:
        *
        '''

        try:
            result = CONSTANTS.FAIL
            response = CONSTANTS.EMPTY_STRING
            result, response = self.obj_comm_lib.readResponse(tool_id, conn_handler, cmd_str)
        except Exception as detail:
            self.logger.logString( MODULE_NAME + ".readResponse : " + str(detail), CONSTANTS.ERROR_LEVEL )
            #self.err_handler.handleError(MODULE_NAME, "Error occured in reading response : " + str(tool_id),str(detail)
        finally:
            return result, response

    def closeConnection( self, tool_id, conn_handler ):
        '''
        * FUNCTION NAME:
        *    closeConnection()
        *
        * DESCRIPTION:
        *    Close connection from TestComplete
        *
        *
        * INPUT:
        *    tool_id        -   tool_id of device
        *    conn_handler   -   connection handler for TestComplete
        *
        * RETURNS:
        *
        * NOTES:
        *
        '''

        try:
            return self.obj_comm_lib.closeConnection(tool_id, conn_handler)
        except Exception as detail:
            self.logger.logString( MODULE_NAME + ".closeConnection : " + str(detail), CONSTANTS.ERROR_LEVEL )
            #self.err_handler.handleError(MODULE_NAME, "Error occured in closing connection : " + str(tool_id),str(detail))


import pandas as pd

class AbstractGrid(object):
    """Grid Builder."""

    def __init__(self):
        """Constructor"""
        self.data_loc = None
        self.interconnect = None
        self.zone2id = {}
        self.id2zone = {}
        self.sub = pd.DataFrame()
        self.plant = pd.DataFrame()
        self.gencost = {"before": pd.DataFrame(), "after": pd.DataFrame()}
        self.dcline = pd.DataFrame()
        self.bus2sub = pd.DataFrame()
        self.bus = pd.DataFrame()
        self.branch = pd.DataFrame()
        self.storage = storage_template()

def storage_template():
    """Get storage

    :return: (*dict*) -- storage structure for MATPOWER/MOST
    """
    storage = {
        "gen": pd.DataFrame(
            columns=[
                "bus_id",
                "Pg",
                "Qg",
                "Qmax",
                "Qmin",
                "Vg",
                "mBase",
                "status",
                "Pmax",
                "Pmin",
                "Pc1",
                "Pc2",
                "Qc1min",
                "Qc1max",
                "Qc2min",
                "Qc2max",
                "ramp_agc",
                "ramp_10",
                "ramp_30",
                "ramp_q",
                "apf",
                "mu_Pmax",
                "mu_Pmin",
                "mu_Qmax",
                "mu_Qmin",
            ]
        ),
        "gencost": pd.DataFrame(
            columns=["type", "startup", "shutdown", "n", "c2", "c1", "c0"]
        ),
        "StorageData": pd.DataFrame(
            columns=[
                "UnitIdx",
                "InitialStorage",
                "InitialStorageLowerBound",
                "InitialStorageUpperBound",
                "InitialStorageCost",
                "TerminalStoragePrice",
                "MinStorageLevel",
                "MaxStorageLevel",
                "OutEff",
                "InEff",
                "LossFactor",
                "rho",
                "ExpectedTerminalStorageMax",
                "ExpectedTerminalStorageMin",
            ]
        ),
        "genfuel": [],
        "duration": None,  # hours
        "min_stor": None,  # ratio
        "max_stor": None,  # ratio
        "InEff": None,
        "OutEff": None,
        "LossFactor": None,  # stored energy fraction / hour
        "energy_price": None,  # $/MWh
        "terminal_min": None,
        "terminal_max": None,
    }
    return storage


#! /usr/local/bin/python

# Joel Schafer: 04-25-06

from DateConvert import MDY,EpochMDY,EpochJul
import GrabList
import sys
from os.path import expanduser

def GetCal(SHEF,JI,Year):

    if isinstance(Year,int):  # ALLOW ENTRY OF '05','2005', OR 2005(INT), BUT NOT 05 (INT)
        tmp= str(Year)
        if len(tmp) == 4:           # INTEGER YEAR MUST BE 4 DIGITS TO CONVERT TO STR PROPERLY
            Year = tmp
        else:
            sys.exit("2 digit integer year, e.g, 02, not allowed in GetSensorCal.py")

    if isinstance(Year,str):    # CONVERT 2-DIGIT STRING YEAR TO 4-DIGIT
        if len(Year) == 2:

            if 90 <= int(Year) <= 99:
                Year= "19" + Year
            else:
                Year= "20" + Year

    Year= int(Year)     # CONVERT BACK TO INT FOR DATECONVERT PROCESS BELOW

    CalFile= "sensors1.fil"

    HomePath= expanduser('~')

    FilePath= HomePath + "/Python/SensorCalFiles/"

    CalFilePath= FilePath + CalFile

    CalSHEFs= GrabList.ColEx(CalFilePath,0,",")
    CalIDs= GrabList.ColEx(CalFilePath,5,",")

    # CREATE LOOKUP DICTIONARY FROM 'sensors1.fil' DATAFILE
    CalDict= dict([(CalSHEFs[i],CalIDs[i]) for i,v in enumerate(CalSHEFs)])

    try:
        ID= CalDict[SHEF].rstrip()  # LOOK-UP DEM ID AND REMOVE TRAILING NEW LINE

#        TmpStr= SHEF + ": " + ID
#        print TmpStr

        IDFile= "calibr" + ID + ".flux"
        IDFilePath= FilePath + IDFile

        CalDay= GrabList.ColEx(IDFilePath,0,":")
        CalMon= GrabList.ColEx(IDFilePath,1,":")
        CalYrAndHr= GrabList.ColEx(IDFilePath,2,":")

        CalVals= GrabList.ColEx(IDFilePath,4,",")

        CalYr= []

        for val in CalYrAndHr:
            tmp= val.split(",")[0]

            if 90 <= int(tmp) <= 99:
                tmp= "19" + tmp
            else:
                tmp= "20" + tmp

            CalYr.append(tmp)

        CalTimes= []

        for i,v in enumerate(CalDay):
            CalTimes.append(EpochMDY(int(CalMon[i]),int(CalDay[i]),int(CalYr[i]),0,0,0))

        SensorCal= CalVals[0]

        CurrTime= EpochJul(Year,JI)

        for i,v in enumerate(CalTimes):
            if CurrTime > CalTimes[i]:
                SensorCal= CalVals[i]
    except:

        SensorCal= 1

    return SensorCal


from dataclasses import dataclass, field
import os
from pathlib import Path
from typing import Any, Dict, List

import numpy as np
from prettytable import PrettyTable
import torch
import tqdm

from .geometry import Projector, gen_probe, feature_pt_ncovis
from .misc import GlobalStepCounter
from .utils import PairwiseCosine

class RecognitionEvaluator():
    def __init__(self, loader, viz=None, writer=None, counter=None, args=None):
        self.viz = viz
        self.probe_pts, self.gds, self.n_observed = None, None, 0
        self.adj_mat = None
        self.loader = loader

        self.writer = writer
        self.counter = counter if counter is not None else GlobalStepCounter()
        self.env_data: Dict[str, _EnvData] = {}
        self.args = args
        self.chunk_size = 128

        self.gt_path = None
        self.eval_gt_dir = args.eval_gt_dir
        if args.eval_gt_dir is not None:
            self.gt_path = Path(args.eval_gt_dir) / f'{args.dataset_name}.npz'

    @torch.no_grad()
    def observe(self, gd, aux, imgs, env_seq):
        B = len(imgs)
        env = env_seq[0][0]
        env_data = self.env_data.setdefault(env, _EnvData(env=env))

        env_data.gds.append(gd)

        if self.args.dataset == 'tartanair':
            seq = env_seq[1][0][0] + '_' + env_seq[1][1][0]
        elif self.args.dataset in ['nordland', 'robotcar']:
            seq = env_seq[1][0]
        # record/update sequence start/end
        seq_lims = env_data.seq_lims.setdefault(seq, [env_data.n_observed, env_data.n_observed + B])
        seq_lims[1] = env_data.n_observed + B

        if self.gt_path is not None:
            if self.args.dataset == 'tartanair':
                depth_map, poses, Ks = aux
                probe_pts = Projector.pix2world(gen_probe(depth_map), depth_map, poses, Ks)
                env_data.aux.extend(probe_pts)
            elif self.args.dataset == 'nordland':
                env_data.aux.extend(aux)
            elif self.args.dataset == 'robotcar':
                env_data.aux.extend(zip(*aux))

        env_data.n_observed += B

    @torch.no_grad()
    def report(self):
        cosine = PairwiseCosine()
        sim_dict = {}
        for env, env_data in self.env_data.items():
            gds = torch.cat(env_data.gds)
            # reduce memory consumption
            cossim = [[cosine(gds[None, st0:nd0], gds[None, st1:nd1])[0].cpu().numpy()
                        for st1, nd1 in chunk_index_itr(len(gds), self.chunk_size)]
                        for st0, nd0 in chunk_index_itr(len(gds), self.chunk_size)]
            sim_dict[env] = np.block(cossim)

        if self.args.eval_save is not None:
            np.savez_compressed(self.args.eval_save, **sim_dict)
            print(f'Saved result: {self.args.eval_save}')

        if self.args.eval_desc_save is not None:
            desc_dict = {env: torch.cat(env_data.gds).cpu().numpy() for env, env_data in self.env_data.items()}
            np.savez_compressed(self.args.eval_desc_save, **desc_dict)
            print(f'Saved global descriptors: {self.args.eval_desc_save}')

        # load or compile groundtruth adjacency
        gt_adj = None
        if self.gt_path.is_file():
            gt_adj = np.load(self.gt_path)
            print(f'Loaded ground truth: {self.gt_path}')
        elif self.eval_gt_dir is not None:
            print('Building groundtruth adjcency matrix')
            env_len, gt_adj = {}, {}
            for env, env_data in self.env_data.items():
                lims = np.array(list(env_data.seq_lims.values()))
                env_len[env] = (lims[:, 1] - lims[:, 0]).sum()
                gt_adj[env] = np.full((env_len[env], env_len[env]), np.nan, dtype=np.float32)

            env_progress = {env: 0 for env in self.env_data}
            for imgs, aux_d, env_seq in tqdm.tqdm(self.loader):
                B, env = len(imgs), env_seq[0][0]
                n_total = env_len[env]
                i = env_progress[env]
                for st, nd in chunk_index_itr(n_total, self.chunk_size):
                    gt_adj[env][st:nd, i:i+B] = self._calc_adjacency(self.env_data[env].aux[st:nd], aux_d)
                env_progress[env] += B

            os.makedirs(self.eval_gt_dir, exist_ok=True)
            np.savez_compressed(self.gt_path, **gt_adj)
            print(f'Saved ground truth: {self.gt_path}')

        if gt_adj is not None:
            table = PrettyTable(field_names=['', 'R@100P'], float_format='.3')
            criterion = get_criterion(self.args.dataset)
            for env in sim_dict:
                gt_adj_ = torch.from_numpy(gt_adj[env]).to(self.args.device).fill_diagonal_(0)
                cossim = torch.from_numpy(sim_dict[env]).to(self.args.device).fill_diagonal_(0)
                r100p = recall_at_100precision(gt_adj_, cossim, criterion)
                table.add_row([env, r100p])
            print(table.get_string())

    def _calc_adjacency(self, aux, aux_d):
        '''Calculate adjacency based on metadata'''
        if self.args.dataset == 'tartanair':
            probe_pts, (depths, poses, K) = aux, aux_d
            probe_pts = torch.stack(probe_pts)
            depths, poses, K = depths.to(probe_pts), poses.to(probe_pts), K.to(probe_pts)

            return feature_pt_ncovis(probe_pts, torch.zeros(len(K)), depths, poses, K).cpu().numpy()
        elif self.args.dataset == 'nordland':
            offset, offset_d = aux, aux_d
            offset, offset_d = torch.stack(offset), offset_d

            return (1 / (np.abs(offset[:, None] - offset_d[None, :]) + 1)).cpu().numpy()
        elif self.args.dataset == 'robotcar':
            HEADING_TOL = 15
            (location, heading), (location_d, heading_d) = list(zip(*aux)), aux_d
            location, heading = torch.stack(location), torch.stack(heading)

            dist = torch.cdist(location, location_d)
            view_diff = (heading[:, None] - heading_d[None, :]).abs()
            return ((view_diff < HEADING_TOL).to(torch.float) / (dist + 1)).cpu().numpy()

@dataclass
class _EnvData():
    env: str
    gds: List[torch.Tensor] = field(default_factory=list)
    n_observed: int = 0
    seq_lims: Dict[str, List[int]] = field(default_factory=dict)
    aux: List[Any] = field(default_factory=list)

def chunk_index_itr(total: int, chunk_size: int):
    for chunk_start in range(0, total, chunk_size):
        chunk_end = min(total, chunk_start + chunk_size)
        yield chunk_start, chunk_end

def get_criterion(dataset_name):
    if dataset_name == 'tartanair':
        return lambda sim: sim > 0.5
    elif dataset_name == 'nordland':
        return lambda sim: sim > 1 / 3.5
    elif dataset_name == 'robotcar':
        return lambda sim: sim > 1 / (10 + 1)

@torch.no_grad()
def recall_at_100precision(gt_sim, pred_sim, relevant):
    pred_rank = pred_sim.sort(dim=1, descending=True).indices
    is_relevant = relevant(gt_sim)

    n_rel = is_relevant.sum(dim=1).to(torch.float32)
    _relevant = is_relevant.gather(1, pred_rank[:, :int(n_rel.max())]).to(torch.float32)
    _n_rec_rel = torch.cumprod(_relevant, dim=1).sum(dim=1)

    return (_n_rec_rel / n_rel.clamp(min=1e-6)).mean().item()


from Basic import TheCoin
from CardIndices import *
import numpy as np

def extractfrom(target, listObject):
	temp = None
	for i in range(len(listObject)):
		if listObject[i] == target:
			temp = listObject.pop(i)
			break #Need this break, so that the iteration won't continue down the shortened list.
	return temp

def fixedList(listObject):
	return listObject[0:len(listObject)]

class Hand_Deck:
	def __init__(self, Game):
		self.Game = Game
		self.hands = {1:[], 2:[]}
		self.decks = {1:[], 2:[]}
		self.noCards = {1:0, 2:0}
		self.handUpperLimit = {1: 10, 2: 10}
		self.initialDecks = {1: WarriorDeck, 2: MageDeck}
		self.startingDeckIdentities = {1:[], 2:[]}
		self.startingHandIdentities = {1:[], 2:[]}
		self.initializeDecks()
		self.initializeHands()

	def initializeDecks(self):
		for ID in range(1, 3):
			Class = self.Game.heroes[ID].Class
			for obj in self.initialDecks[ID]:
				card = obj(self.Game, ID)
				if "Galakrond. " in card.name:
					#检测过程中，如果目前没有主迦拉克隆或者与之前检测到的迦拉克隆与玩家的职业不符合，则把检测到的迦拉克隆定为主迦拉克隆
					if self.Game.CounterHandler.primaryGalakronds[ID] == None:
						self.Game.CounterHandler.primaryGalakronds[ID] = card
					elif self.Game.CounterHandler.primaryGalakronds[ID].Class != Class and card.Class == Class:
						self.Game.CounterHandler.primaryGalakronds[ID] = card
				card.entersDeck()
				self.decks[ID].append(card)
				self.startingDeckIdentities[ID].append(card.identity)
			np.random.shuffle(self.decks[ID])

	def initializeHands(self):
		#如果卡组有双传说任务，则起手时都会上手
		mainQuests = {1:[], 2:[]}
		mulliganSize = {1:3, 2:4}
		for ID in range(1, 3):
			for card in self.decks[ID]:
				if card.description.startswith("Quest: "):
					mainQuests[ID].append(card)
			numQueststoDraw = min(len(mainQuests[ID]), mulliganSize[ID])
			if numQueststoDraw > 0:
				queststoDraw = self.extractfromDeck(np.random.choice(mainQuests[ID], numQueststoDraw, replace=False))[0]
				for quest in queststoDraw:
					self.Game.mulligans[ID].append(quest)
			for i in range(mulliganSize[ID]-numQueststoDraw):
				self.Game.mulligans[ID].append(self.extractfromDeck(self.decks[ID][-1])[0])

	def mulligan(self, indicesCards1, indicesCards2):
		indicesCards = {1:indicesCards1, 2:indicesCards2}
		print("Player 1's cards to replace are", indicesCards[1], "Player 2's cards to replace are", indicesCards[2])
		for ID in range(1, 3):
			cardstoReplace = []
			#self.Game.mulligans is the cards currently in players' hands.
			if indicesCards[ID] != []:
				for num in range(1, len(indicesCards[ID])+1):
					cardstoReplace.append(self.Game.mulligans[ID].pop(indicesCards[ID][-num]))
				#调用手牌中没有被替代的卡的entersHand()
				for card in self.Game.mulligans[ID]:
					card = card.entersHand()
				self.hands[ID] = self.Game.mulligans[ID]

				for i in range(len(indicesCards[ID])):
					self.drawCard(ID)
				self.decks[ID] += cardstoReplace
				np.random.shuffle(self.decks[ID]) #Shuffle the deck after mulligan
			else: #No card replaced
				for card in self.Game.mulligans[ID]:
					card.leavesDeck()
					card = card.entersHand()
				self.hands[ID] = self.Game.mulligans[ID]

			#Record the starting hand identities.
			for card in self.hands[ID]:
				self.startingHandIdentities[ID].append(card.identity)

			print("Player's starting hand:", self.hands[ID])

		self.addCardtoHand(TheCoin(self.Game, 2), 2)
		self.Game.ManaHandler.calcMana_All()
		for ID in range(1, 3):
			for card in self.hands[ID] + self.decks[ID]:
				if "Start of Game" in card.index:
					card.startofGame()

	def handNotFull(self, ID):
		if len(self.hands[ID]) < self.handUpperLimit[ID]:
			return True
		return False

	def spaceinHand(self, ID):
		return self.handUpperLimit[ID] - len(self.hands[ID])

	def noDuplicatesinDeck(self, ID):
		record = []
		for card in self.decks[ID]:
			if type(card) not in record:
				record.append(type(card))
			else:
				print("The record is ", record)
				return False
		return True

	def holdingDragon(self, ID, minion=None):
		if minion == None: #When card not in hand and wants to check if a Dragon is in hand
			for card in self.hands[ID]:
				if card.cardType == "Minion" and "Dragon" in card.race:
					return True
		else: #When the minion is inHand and wants to know if it can trigger after being played.
			for card in self.hands[ID]:
				if card.cardType == "Minion" and "Dragon" in card.race and card != minion:
					return True
		return False

	def holdingSpellwith5CostorMore(self, ID):
		for card in self.hands[ID]:
			if card.cardType == "Spell" and card.mana >= 5:
				return True
		return False

	def holdingCardfromAnotherClass(self, ID, card=None):
		Class = self.Game.heroes[ID].Class
		if card == None:
			for cardinHand in self.hands[ID]:
				if Class not in cardinHand.Class and cardinHand.Class != "Neutral":
					return True
			return False
		else:
			for cardinHand in self.hands[ID]:
				if Class not in cardinHand.Class and cardinHand.Class != "Neutral" and cardinHand != card:
					return True
			return False

	#抽牌一次只能一张，需要废除一次抽多张牌的功能，因为这个功能都是用于抽效果指定的牌。但是这些牌中如果有抽到时触发的技能，可能会导致马上抽牌把列表后面的牌提前抽上来
	#现在规则为如果要连续抽2张法术，则分两次检测牌库中的法术牌，然后随机抽一张。
	#如果这个规则是正确的，则在牌库只有一张夺灵者哈卡的堕落之血时，抽到这个法术之后会立即额外抽牌，然后再塞进去两张堕落之血，那么第二次抽法术可能会抽到新洗进去的堕落之血。
	#Damage taken due to running out of card will keep increasing. Refilling the deck won't reset the damage you take next time you draw from empty deck
	def drawCard(self, ID, card=None):
		if card == None: #Draw from top of the deck.
			print("Hero %d draws from the top of the deck"%ID)
			if self.decks[ID] == []: #No cards left in deck.
				print("Hero%d's"%ID + "deck is empty and will take damage")
				self.noCards[ID] += 1 #如果在疲劳状态有卡洗入牌库，则疲劳值不会减少，在下次疲劳时，仍会从当前的非零疲劳值开始。
				damage = self.noCards[ID]
				objtoTakeDamage = self.Game.DamageHandler.damageTransfer(self.Game.heroes[ID])
				targetSurvival = objtoTakeDamage.damageRequest(None, damage)
				if targetSurvival > 0:
					damageActual, survival = objtoTakeDamage.takesDamage(None, damage) #疲劳伤害没有来源
				return (None, 0)
			else:
				card = self.decks[ID].pop()
				mana = card.mana
		else:
			print("Hero %d draws %s from the deck"%(ID, card.name))
			card = extractfrom(card, self.decks[ID])
			mana = card.mana
		card.leavesDeck()
		if self.handNotFull(ID):
			cardTracker = [card] #把这张卡放入一个列表，然后抽牌扳机可以对这个列表进行处理同时传递给其他抽牌扳机
			self.Game.sendSignal("CardDrawn", ID, None, cardTracker, mana, "")
			if cardTracker[0].cardType == "Spell" and "Casts When Drawn" in cardTracker[0].index:
				print(cardTracker[0].name, " is drawn and cast.")
				cardTracker[0].whenEffective()
				self.drawCard(ID)
				cardTracker[0].afterDrawingCard()
			else: #抽到的牌可以加入手牌。
				if cardTracker[0].cardType == "Minion" and "Triggers when Drawn" in cardTracker[0].index:
					print(cardTracker[0].name, " is drawn and triggers its effect.")
					for func in cardTracker[0].triggers["Drawn"]:
						func()
				cardTracker[0] = cardTracker[0].entersHand()
				self.hands[ID].append(cardTracker[0])
				self.Game.sendSignal("CardEntersHand", ID, None, cardTracker, mana, "")
				self.Game.ManaHandler.calcMana_All()
			return (cardTracker[0], mana)
		else:
			print("Player's hand is full. The drawn card %s is milled"%card.name)
			return (None, 0)

	#Will force the ID of the card to change.
	def addCardtoHand(self, obj, ID, comment="AddRealCard", index=-1):
		if type(obj) == type([]) or type(obj) == type(np.array([])): #Multiple cards at a time
			for card in obj:
				if self.handNotFull(ID):
					if comment == "CreateUsingIndex":
						card = self.Game.cardPool[card](self.Game, ID)
					elif comment == "CreateUsingType":
						card = card(self.Game, ID)

					card.ID = ID
					card = card.entersHand()
					#Add the card to hand.
					if index == -1:
						self.hands[ID].append(card)
					else:
						self.hands[ID].insert(index, card)
					print(card.name, " is put into player %d's hand."%ID)
					self.Game.sendSignal("CardEntersHand", ID, None, [card], 0, comment)
				else:
					print("Player's hand is full. Can't add more cards.")
					break
		else: #If the obj is a single card/index/type.
			if self.handNotFull(ID):
				if comment == "CreateUsingIndex":
					obj = self.Game.cardPool[obj](self.Game, ID)
				elif comment == "CreateUsingType":
					obj = obj(self.Game, ID)
				obj.ID = ID
				obj = obj.entersHand()
				#Add card into hand.
				if index == -1:
					self.hands[ID].append(obj)
				else:
					self.hands[ID].insert(index, obj)
				#Process the card's entersHand() method.
				print(obj.name, " is added into player %d's hand."%ID)
				self.Game.sendSignal("CardEntersHand", ID, None, [obj], 0, comment)
			else:
				print("Player's hand is full. Can't add more cards.")

		self.Game.ManaHandler.calcMana_All()

	def replaceCardDrawn(self, targetHolder, newCard):
		ID = targetHolder[0].ID
		isPrimaryGalakrond = targetHolder[0] == self.Game.CounterHandler.primaryGalakronds[ID]
		targetHolder[0] = newCard
		if isPrimaryGalakrond:
			self.Game.CounterHandler.primaryGalakronds[ID] = newCard

	def replaceCardinHand(self, card, newCard):
		ID = card.ID
		for i in range(len(self.hands[ID])):
			if self.hands[ID][i] == card:
				card.leavesHand()
				self.hands[ID].pop(i)
				self.Game.sendSignal("CardLeavesHand", ID, None, card, 0, "")
				self.addCardtoHand(newCard, ID, "AddRealCard", i)
				break

	def replaceCardinDeck(self, card, newCard):
		ID = card.ID
		for i in range(len(self.decks[ID])):
			if self.decks[ID][i] == card:
				card.leavesDeck()
				self.decks[ID].pop(i)
				newCard.entersDeck()
				self.decks[ID].insert(i, newCard)
				break

	#All the cards shuffled will be into the same deck. If necessary, invoke this function for each deck.
	#PlotTwist把手牌洗入牌库的时候，手牌中buff的随从两次被抽上来时buff没有了。
	#假设洗入牌库这个动作会把一张牌初始化
	def shuffleCardintoDeck(self, obj, InitiatorID):
		if type(obj) == type([]) or type(obj) == type(np.array([])):
			ID = obj[0].ID
			for card in obj:
				self.decks[ID].append(card)
				card.entersDeck()
		else: #Shuffle a single card
			ID = obj.ID
			self.decks[ID].append(obj)
			obj.entersDeck()

		self.Game.sendSignal("CardShuffled", InitiatorID, None, obj, 0, "")
		np.random.shuffle(self.decks[ID])

	def discardCard(self, ID, card=None, discardAll=False):
		if discardAll: #Discard all hand.
			if self.hands[ID] != []:
				cards, cost, isRightmostCardinHand = self.extractfromHand(None, all=True, ID=ID)
				for card in cards:
					print("Card %s in player's hand is discarded:"%card.name)
					for func in card.triggers["Discarded"]:
						func()
					self.Game.CounterHandler.cardsDiscardedThisGame[ID].append(card.index)
					self.Game.sendSignal("PlayerDiscardsCard", card.ID, None, card, 0, "")
				self.Game.ManaHandler.calcMana_All()
		else: #Discard a single card.
			if card == None: #Discard a random card.
				if self.hands[ID] != []:
					card = np.random.choice(self.hands[ID])
					card, cost, isRightmostCardinHand = self.extractfromHand(card)
					print("Card %s in player's hand is discarded:"%card.name)
					for func in card.triggers["Discarded"]:
						func()
					self.Game.ManaHandler.calcMana_All()
					self.Game.CounterHandler.cardsDiscardedThisGame[ID].append(card.index)
					self.Game.sendSignal("CardLeavesHand", card.ID, None, card, 0, "")
					self.Game.sendSignal("PlayerDiscardsCard", card.ID, None, card, 0, "")
			else: #Discard a chosen card.
				card, cost, isRightmostCardinHand = self.extractfromHand(card)
				print("Card %s in player's hand is discarded:"%card.name)
				for func in card.triggers["Discarded"]:
					func()
				self.Game.ManaHandler.calcMana_All()
				self.Game.CounterHandler.cardsDiscardedThisGame[ID].append(card.index)
				self.Game.sendSignal("CardLeavesHand", card.ID, None, card, 0, "")
				self.Game.sendSignal("PlayerDiscardsCard", card.ID, None, card, 0, "")

	def extractfromHand(self, card, all=False, ID=0):
		if all: #Extract the entire hand.
			temp = self.hands[ID]
			self.hands[ID] = []
			for card in temp:
				card.leavesHand()
				self.Game.sendSignal("CardLeavesHand", card.ID, None, card, 0, '')
			return temp, 0, False
		else:
			if type(card) != type([]) and type(card) != type(np.array([])): #Extracting a single card from hand.
				#Need to keep track of the card's location in hand.
				for i in range(len(self.hands[card.ID])):
					if self.hands[card.ID][i] == card:
						index, cost = i, card.mana
						isRightmostCardinHand = True if i == len(self.hands[card.ID]) - 1 else False
						break
				card = self.hands[card.ID].pop(index)
				card.leavesHand()
				self.Game.sendSignal("CardLeavesHand", card.ID, None, card, 0, '')
				return card, cost, isRightmostCardinHand
			else: #Extracting multiple cards from hand.
				ID = card[0].ID #Will only invoke this function for one side. If necessary, invoke once for each hand.
				cards = []
				for obj in card:
					result = extractfrom(obj, self.hands[ID])
					if result != None:
						result.leavesHand()
						cards.append(result)
						self.Game.sendSignal("CardLeavesHand", card.ID, None, card, 0, '')
				return cards, 0, False

	def extractfromDeck(self, card, all=False, ID=0):
		if all: #For replacing the entire deck or throwing it away.
			temp = self.decks[ID]
			self.decks[ID] = []
			for card in temp:
				card.leavesDeck()
			return temp, 0, False
		else:
			if type(card) != type([]) and type(card) != type(np.array([])): #Extracting a single card from deck.
				card = extractfrom(card, self.decks[card.ID])
				card.leavesDeck()
				return card, 0, False
			else: #Extracting multiple cards from hand.
				ID = card[0].ID
				cards = []
				for obj in card:
					result = extractfrom(obj, self.decks[ID])
					result.leavesDeck()
					cards.append(result)
				return cards, 0, False

	def removeDeckTopCard(self, ID):
		if self.decks[ID] != []:
			card = self.decks[ID].pop(0)
			card.leavesDeck()
			print("The top card %s in player %d's deck is removed"%(card.name, ID))
			return card
		else:
			return None

DruidDeck = [Cenarius, PoweroftheWild, WardruidLoti, FlobbidinousFloop, UntappedPotential, YseraUnleashed, Swipe, Swipe, Starfall, Starfall, Nourish, Nourish, Innervate, Innervate, Wrath, Wrath, CrystalMerchant, CrystalMerchant, HiddenOasis, HiddenOasis, WorthyExpedition, WorthyExpedition, AnubisathDefender, AnubisathDefender, OasisSurger, OasisSurger, RisingWinds, RisingWinds, SteelBeetle, SteelBeetle, ]

HunterDeck = [LeeroyJenkins, CultMaster, BoommasterFlark, Zilliax, HalazzitheLynx, UnsealtheVault, BoneWraith, FacelessCorruptor, Veranus, SN1PSN4P, Tracking, Tracking, UnleashtheHounds, UnleashtheHounds, Springpaw, Springpaw, HenchClanHogsteed, HenchClanHogsteed, DesertSpear, DesertSpear, QuestingExplorer, QuestingExplorer, SwarmofLocusts, SwarmofLocusts, DivingGryphon, DivingGryphon, CleartheWay, CleartheWay, LicensedAdventurer, LicensedAdventurer, ]

MageDeck = [Doomsayer, Blizzard, ArcaneIntellect, Alexstrasza, FrostNova, MountainGiant, TwilightDrake, ArcaneKeysmith, BookofSpecters, StargazerLuna, LunasPocketGalaxy, Zilliax, FiretreeWitchdoctor, ConjurersCalling, RayofFrost, PowerofCreation, Kalecgos, KhartutDefender, ZephrystheGreat, TortollanPilgrim, RenotheRelicologist, BoneWraith, Siamat, ArcaneBreath, Dragoncaster, DragonqueenAlexstrasza, EscapedManasaber, TheAmazingReno, MalygosAspectofMagic, SN1PSN4P, ]

PaladinDeck = [LeeroyJenkins, BlessingofKings, Zilliax, SN1PSN4P, TruesilverChampion, TruesilverChampion, ReplicatingMenace, ReplicatingMenace, Mecharoo, Mecharoo, GlowTron, GlowTron, Galvanizer, Galvanizer, Crystology, Crystology, AnnoyoModule, AnnoyoModule, Wargear, Wargear, MicroMummy, MicroMummy, SkyClaw, SkyClaw, HotAirBalloon, HotAirBalloon, GoboglideTech, GoboglideTech, Shotbot, Shotbot, ]

PriestDeck = [WildPyromancer, Silence, AcolyteofPain, ExtraArms, TopsyTurvy, BwonsamditheDead, HighPriestAmet, DarkProphecy, InnerFire, InnerFire, PowerWordShield, PowerWordShield, InjuredBlademaster, InjuredBlademaster, DivineSpirit, DivineSpirit, CircleofHealing, CircleofHealing, NorthshireCleric, NorthshireCleric, Lightwarden, Lightwarden, Psychopomp, Psychopomp, InjuredTolvir, InjuredTolvir, BeamingSidekick, BeamingSidekick, NefersetRitualist, NefersetRitualist, ]

RogueDeck = [Backstab, EdwinVanCleef, Shadowstep, Sap, LeeroyJenkins, Eviscerate, SI7Agent, Zilliax, EVILMiscreant, HeistbaronTogwaggle, UnderbellyFence, Vendetta, ZephrystheGreat, BoneWraith, Siamat, PharaohCat, DragonsHoard, PraiseGalakrond, BloodsailFlybooter, FlikSkyshiv, ShieldofGalakrond, DevotedManiac, SealFate, FacelessCorruptor, DragonqueenAlexstrasza, KronxDragonhoof, EscapedManasaber, BoompistolBully, SN1PSN4P, GalakrondtheNightmare, ]

ShamanDeck = [EarthShock, EarthenMight, Shudderwock, ElectraStormsurge, HauntingVisions, Zentimo, KronxDragonhoof, GalakrondtheTempest, Hex, Hex, FarSight, FarSight, TotemicSmash, TotemicSmash, SpiritoftheFrog, SpiritoftheFrog, Mutate, Mutate, MoguFleshshaper, MoguFleshshaper, DragonsPack, DragonsPack, CorruptElementalist, CorruptElementalist, ShieldofGalakrond, ShieldofGalakrond, DevotedManiac, DevotedManiac, InvocationofFrost, InvocationofFrost, ]

WarlockDeck = [Alexstrasza, LordGodfrey, Zilliax, ZephrystheGreat, DragonqueenAlexstrasza, KronxDragonhoof, SN1PSN4P, GalakrondtheWretched, SacrificialPact, SacrificialPact, MortalCoil, MortalCoil, PlagueofFlames, PlagueofFlames, CrazedNetherwing, CrazedNetherwing, DragonblightCultist, DragonblightCultist, VeiledWorshipper, VeiledWorshipper, DarkSkies, DarkSkies, NetherBreath, NetherBreath, ShieldofGalakrond, ShieldofGalakrond, DevotedManiac, DevotedManiac, BadLuckAlbatross, BadLuckAlbatross, ]

WarriorDeck = [BattleRage, LeeroyJenkins, KronxDragonhoof, BombWrangler, SN1PSN4P, GalakrondtheUnbreakable, InnerRage, InnerRage, Armorsmith, Armorsmith, AcolyteofPain, AcolyteofPain, TownCrier, TownCrier, EterniumRover, EterniumRover, BloodswornMercenary, BloodswornMercenary, ScionofRuin, ScionofRuin, RitualChopper, RitualChopper, ShieldofGalakrond, ShieldofGalakrond, DevotedManiac, DevotedManiac, Awaken, Awaken, RiskySkipper, RiskySkipper, ]


#!/usr/bin/env python3
# -*- coding: UTF-8 -*-

import cv2
import time
import math
import numpy as np
import tensorflow as tf
from OneEuroFilter import OneEuroFilter

class VNectEstimator:
    # the side length of the bounding box
    _box_size = 368
    # this factor indicates that the input box size is 8 times the side length of the output heatmaps
    _hm_factor = 8
    # number of the joints to be detected
    _joints_num = 21

    def __init__(self, T=False):
        print('Initializing VnectEstimator...')
        # whether apply transposed matrix (when camera is flipped)
        self.T = T
        # the ratio factors to scale the input image crops, no more than 1.0
        self.scales = [1]  # or [1, 0.7] to be consistent with the author when training
        # initialize one euro filters for all the joints
        config_2d = {
            'freq': 12,
            'mincutoff': 1.7,
            'beta': 0.3,
            'dcutoff': 1.0
        }
        config_3d = {
            'freq': 12,
            'mincutoff': 0.8,
            'beta': 0.4,
            'dcutoff': 1.0
        }
        self.filter_2d = [(OneEuroFilter(**config_2d), OneEuroFilter(**config_2d)) for _ in range(self._joints_num)]
        self.filter_3d = [(OneEuroFilter(**config_3d), OneEuroFilter(**config_3d), OneEuroFilter(**config_3d))
                          for _ in range(self._joints_num)]
        # load pretrained VNect model
        self.sess = tf.Session()
        saver = tf.train.import_meta_graph('./models/tf_model/vnect_tf.meta')
        saver.restore(self.sess, tf.train.latest_checkpoint('./models/tf_model/'))
        graph = tf.get_default_graph()
        self.input_crops = graph.get_tensor_by_name('Placeholder:0')
        self.heatmap = graph.get_tensor_by_name('split_2:0')
        self.x_heatmap = graph.get_tensor_by_name('split_2:1')
        self.y_heatmap = graph.get_tensor_by_name('split_2:2')
        self.z_heatmap = graph.get_tensor_by_name('split_2:3')
        print('Initialization done.')

    @staticmethod
    def _gen_input_batch(img_input, box_size, scales):
        # any input image --> sqrared input image acceptable for the model
        img_square = img_scale_squareify(img_input, box_size)
        # generate multi-scale input batch
        input_batch = []
        for scale in scales:
            img = img_scale_padding(img_square, scale) if scale < 1 else img_square
            input_batch.append(img)
        # input image range: [0, 255) --> [-0.4, 0.6)
        input_batch = np.asarray(input_batch, dtype=np.float32) / 255 - 0.4
        return input_batch

    def _joint_filter(self, joints_2d, joints_3d):
        for i in range(self._joints_num):
            joints_2d[i, 0] = self.filter_2d[i][0](joints_2d[i, 0], time.time())
            joints_2d[i, 1] = self.filter_2d[i][1](joints_2d[i, 1], time.time())

            joints_3d[i, 0] = self.filter_3d[i][0](joints_3d[i, 0], time.time())
            joints_3d[i, 1] = self.filter_3d[i][1](joints_3d[i, 1], time.time())
            joints_3d[i, 2] = self.filter_3d[i][2](joints_3d[i, 2], time.time())
        return joints_2d, joints_3d

    def __call__(self, img_input):
        t = time.time()
        img_input = img_input.T if self.T else img_input
        img_batch = self._gen_input_batch(img_input, self._box_size, self.scales)
        # inference
        hm, xm, ym, zm = self.sess.run([self.heatmap, self.x_heatmap, self.y_heatmap, self.z_heatmap],
                                       {self.input_crops: img_batch})
        # average scale outputs
        hm_size = self._box_size // self._hm_factor
        hm_avg = np.zeros((hm_size, hm_size, self._joints_num))
        xm_avg = np.zeros((hm_size, hm_size, self._joints_num))
        ym_avg = np.zeros((hm_size, hm_size, self._joints_num))
        zm_avg = np.zeros((hm_size, hm_size, self._joints_num))
        for i in range(len(self.scales)):
            rescale = 1.0 / self.scales[i]
            scaled_hm = img_scale(hm[i, :, :, :], rescale)
            scaled_x_hm = img_scale(xm[i, :, :, :], rescale)
            scaled_y_hm = img_scale(ym[i, :, :, :], rescale)
            scaled_z_hm = img_scale(zm[i, :, :, :], rescale)
            mid = [scaled_hm.shape[0] // 2, scaled_hm.shape[1] // 2]
            hm_avg += scaled_hm[mid[0] - hm_size // 2: mid[0] + hm_size // 2,
                                mid[1] - hm_size // 2: mid[1] + hm_size // 2, :]
            xm_avg += scaled_x_hm[mid[0] - hm_size // 2: mid[0] + hm_size // 2,
                                  mid[1] - hm_size // 2: mid[1] + hm_size // 2, :]
            ym_avg += scaled_y_hm[mid[0] - hm_size // 2: mid[0] + hm_size // 2,
                                  mid[1] - hm_size // 2: mid[1] + hm_size // 2, :]
            zm_avg += scaled_z_hm[mid[0] - hm_size // 2: mid[0] + hm_size // 2,
                                  mid[1] - hm_size // 2: mid[1] + hm_size // 2, :]
        hm_avg /= len(self.scales)
        xm_avg /= len(self.scales)
        ym_avg /= len(self.scales)
        zm_avg /= len(self.scales)
        joints_2d = extract_2d_joints_from_heatmap(hm_avg, self._box_size, self._hm_factor)
        joints_3d = extract_3d_joints_from_heatmap(joints_2d, xm_avg, ym_avg, zm_avg, self._box_size, self._hm_factor)
        joints_2d, joints_3d = self._joint_filter(joints_2d, joints_3d)
        print('FPS: {:>2.2f}'.format(1 / (time.time() - t)))
        return joints_2d, joints_3d

def img_scale(img, scale):
    """
    scale the input image by a same scale factor in both x and y directions
    """
    return cv2.resize(img, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_LANCZOS4)

def img_padding(img, box_size, offset, padNum=0):
    """
    pad the image in left and right sides averagely to fill the box size

    padNum: the number to be filled (0-->(0, 0, 0)==black; 128-->(128, 128, 128)==grey)
    """
    h, w = img.shape[:2]
    assert h == box_size, 'height of the image not equal to box size'
    assert w < box_size, 'width of the image not smaller than box size'

    img_padded = np.ones((box_size, box_size, 3), dtype=np.uint8) * padNum
    img_padded[:, box_size // 2 - math.ceil(img.shape[1] / 2):box_size // 2 + math.ceil(img.shape[1] / 2) - offset,
               :] = img

    return img_padded

def img_scale_squareify(img, box_size):
    """
    scale and squareify the image to get a square image with standard box size

    img: BGR image
    boxsize: the length of the square area
    """
    h, w = img.shape[:2]
    scale = box_size / h
    img_scaled = img_scale(img, scale)

    if img_scaled.shape[1] < box_size:
        offset = img_scaled.shape[1] % 2
        img_cropped = img_padding(img_scaled, box_size, offset)
    else:
        img_cropped = img_scaled[:, img_scaled.shape[1] // 2 - box_size // 2:img_scaled.shape[1] // 2 + box_size // 2,
                                 :]

    assert img_cropped.shape == (box_size, box_size, 3)
    return img_cropped

def img_scale_padding(img, scale, padNum=0):
    """
    scale and pad the image

    scale: no bigger than 1.0
    """
    assert img.shape[0] == img.shape[1]
    box_size = img.shape[0]
    img_scaled = img_scale(img, scale)
    pad_h = (box_size - img_scaled.shape[0]) // 2
    pad_w = (box_size - img_scaled.shape[1]) // 2
    pad_h_offset = (box_size - img_scaled.shape[0]) % 2
    pad_w_offset = (box_size - img_scaled.shape[1]) % 2
    img_scaled_padded = np.pad(img_scaled, ((pad_w, pad_w + pad_w_offset), (pad_h, pad_h + pad_h_offset), (0, 0)),
                               mode='constant', constant_values=padNum)

    return img_scaled_padded

def extract_2d_joints_from_heatmap(heatmap, box_size, hm_factor):
    """
    rescale the heatmap to CNN input size, then record the coordinates of each joints

    joints_2d: a joints_num x 2 array, each row contains [row, column] coordinates of the corresponding joint
    """
    assert heatmap.shape[0] == heatmap.shape[1]
    heatmap_scaled = img_scale(heatmap, hm_factor)

    joints_2d = np.zeros((heatmap_scaled.shape[2], 2), dtype=np.int16)
    for joint_num in range(heatmap_scaled.shape[2]):
        joint_coord = np.unravel_index(np.argmax(heatmap_scaled[:, :, joint_num]), (box_size, box_size))
        joints_2d[joint_num, :] = joint_coord

    return joints_2d

def extract_3d_joints_from_heatmap(joints_2d, x_hm, y_hm, z_hm, box_size, hm_factor):
    """
    obtain the 3D coordinates of each joint from its 2D coordinates

    joints_3d: a joints_num x 3 array, each row contains [x, y, z] coordinates of the corresponding joint

    Notation:
    x direction: left --> right
    y direction: up --> down
    z direction: forawrd --> backward
    """
    scaler = 100
    joints_3d = np.zeros((x_hm.shape[2], 3), dtype=np.float32)

    for joint_num in range(x_hm.shape[2]):
        coord_2d_h, coord_2d_w = joints_2d[joint_num][:]

        joint_x = x_hm[max(int(coord_2d_h / hm_factor), 1), max(int(coord_2d_w / hm_factor), 1), joint_num] * scaler
        joint_y = y_hm[max(int(coord_2d_h / hm_factor), 1), max(int(coord_2d_w / hm_factor), 1), joint_num] * scaler
        joint_z = z_hm[max(int(coord_2d_h / hm_factor), 1), max(int(coord_2d_w / hm_factor), 1), joint_num] * scaler
        joints_3d[joint_num, :] = [joint_x, joint_y, joint_z]

    # Subtract the root location to normalize the data
    joints_3d -= joints_3d[14, :]

    return joints_3d

if __name__ == '__main__':
    estimator = VNectEstimator()
    joints_2d, joints_3d = estimator('some pic')
    print(joints_2d, '\n', joints_3d)


from django.conf.urls import patterns, url

from .views import flat_page_view

urlpatterns = patterns('',
    url(
        regex=r'^how_to_adopt/$',
        view=flat_page_view,
        name='flat_how_to_adopt',
        kwargs={'template_name': 'flat/how_to_adopt.html'},
    ),

    url(
        regex=r'^how_to_foster/$',
        view=flat_page_view,
        name='flat_how_to_foster',
        kwargs={'template_name': 'flat/how_to_foster.html'},
    ),

    url(
        regex=r'^programs/$',
        view=flat_page_view,
        name='flat_programs',
        kwargs={'template_name': 'flat/programs.html'},
    ),

    url(
        regex=r'^about/$',
        view=flat_page_view,
        name='flat_about',
        kwargs={'template_name': 'flat/about.html'},
    ),
)


import math
# outputs the sqrt of numbers between 1 & n in a table format
def sqrtTable(n):
    result=1
    print ("Number \t\t\t Square root")
    for i in range(0,n+1):
       print (i,end="\t\t\t")
       print (math.sqrt(i))

# the main function
def main():
    num = eval(input("Enter the number to print sqrt table:\n"))
    sqrtTable(num)

answer="Y"
while (answer=="Y" or answer=="y"):
    main() # Call the main function
    answer= input("Do you want to repeat (Y|y)?")


# coding:utf-8
# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/topics/items.html

from scrapy.item import Item, Field
from scrapy.loader import ItemLoader
from scrapy.loader.processors import MapCompose, TakeFirst, Join

class JuziItem(Item):

    info_id = Field()
    company_name = Field()
    slogan = Field()
    sub_scope = Field()
    city = Field()
    area = Field()
    home_page = Field()
    tags = Field()
    company_intro = Field()
    company_full_name = Field()
    found_time = Field()
    company_size = Field()
    company_status = Field()
    tz_info = Field()
    tm_info = Field()
    pdt_info = Field()

class ExampleLoader(ItemLoader):
    default_item_class = JuziItem
    default_input_processor = MapCompose(lambda s: s.strip())
    default_output_processor = TakeFirst()
    description_out = Join()


from flask_mysqldb import MySQL
from app import mysql

def getOneDataFromDatabase(query, param = None):
    cur = mysql.connection.cursor()
    cur.execute(query, param)
    data = cur.fetchone()
    cur.close()
    return data

def getDataFromDatabase(query, param = None):
    cur = mysql.connection.cursor()
    cur.execute(query, param)
    data = cur.fetchall()
    cur.close()
    return data

def updateDatabase(query, param):
    cur = mysql.connection.cursor()
    cur.execute(query, param)
    mysql.connection.commit()
    cur.close()
    return ""

def insertIntoDatabase(query, param):
    cur = mysql.connection.cursor()
    cur.execute(query, param)
    mysql.connection.commit()
    cur.close()
    return ""

def deleteFromDatabase(query, param):
    cur = mysql.connection.cursor()
    if "ugroup" in query:
        updateDatabase("UPDATE users set ugroup = '*deleted user group*' WHERE ugroup = %s", [param])
    elif "lcategory" in query:
        updateDatabase("UPDATE users set lcat = '*deleted leave category*' WHERE lcat = %s", [param])

    cur.execute(query, param)
    mysql.connection.commit()
    cur.close()
    return ""