import codecs;
import random;
import re;
import glob;
import os;

import math
import re
import sys

import json

# add hyper paramter if unbalanced
hyper_parameter = 200

# random seed
# and inner loop random seed change
rand_seed_parameter_initialization = 0.1
rand_seed_offset = 0.01

fileDomainRelatedIntent = ['file_search', 'file_open', 'file_share', 'file_download', 'file_other', 'file_navigate', "teamspace_search"]

teamsDomainToFileDomain = {
    # carina need small but bellevue needs big
    "teams" : "files",
    "TEAMS" : "FILES"
}

defaultFileTypeifMissed =[
        'document',
	'documents',
	'file',
	'files',
	'powerpoints',
	'power point',
	'slide',
	'slides',
	'doc',
	'docx',
	'docs',
	'spec',
	'excel',
	'excels',
	'xls',
	'xlsx',
	'spreadsheet',
	'spreadsheets',
	'workbook',
	'worksheet',
	'csv',
	'tsv',
	'note',
	'notes',
	'onenote',
	'onenotes',
	'onenote',
	'notebook',
	'notebooks',
	'pdf',
	'pdfs',
	'pdf',
	'jpg',
	'jpeg',
	'gif',
	'png',
	'image',
	'msg',
	'ics',
	'vcs',
	'vsdx',
	'vssx',
	'vstx',
	'vsdm',
	'vssm',
	'vstm',
	'vsd',
	'vdw',
	'vss',
	'vst',
	'mpp',
	'mpt',
	'word',
	'words'
#    'file',
#    'files',
#    'document',
#    'documents'
]

# for o(1) loook up
defaultFileTypeifMissedSet = set(defaultFileType.lower() for defaultFileType in defaultFileTypeifMissed)

outputs = [];
outputsSet = set([]);

outputsWithSource = [];

outputFileKeyWordAndFileNameUserContext = set([]);

fileKeyWordAndFileNameCandidateSet = set()

OutputSlotEvaluation = [];

OutputIntentEvaluation = [];

OutputSTCAIntentEvaluation = [];

# leave it but actually it is not being used
dsatTraining = "dsat_training.tsv"

outputFile = 'contexual_filekeyword_filetype_training.tsv'

outputFileUnique = 'contexual_filekeyword_filetype_unique_training.tsv'
# replace directly
#outputTrainingFolderFile = '..\\files_slot_training.tsv'
# for STCA test
#outputSTCATrainingFolderFile = '..\\sharemodeltest\\files_slot_training.tsv'

outputFileWithSource = "contexual_filekeyword_filetype_training_with_source.tsv"

outputFileKeyWordAndFileNameLexiconFilfe = 'contexual_filekeyword_filename_lexicon.txt'

outputFileKeyWordAndFileNameUserContextFiles = 'user_filekeyword_filename.txt'

# validaitng opened dataset
# old format work
inputFile = "files_file_keyword_positive.tsv"

#initial rand seed
rand_seed_parameter = rand_seed_parameter_initialization

ConversationIdPrefix = 'contexual_filekeyword_fileType'
ConversationId = 0;

#with codecs.open('Teams-golden.tsv', 'r', 'utf-8') as fin:
with codecs.open(inputFile, 'r', 'utf-8') as fin:

    for line in fin:
        line = line.strip();
        if not line:
            continue;
        array = line.split('\t');

        # make sure it at least has
        # Query	ExternalFeature	Weight	Intent	Domain	Slot
        if len(array) < 6:
            continue;

        if array[1] != 'FILES':
            continue

        # update seed for each query
        rand_seed_parameter=rand_seed_parameter+rand_seed_offset;
        random.seed(rand_seed_parameter);

        #add new here
        slot = array[3]
        # for extra target slot you want
        xmlpairs = re.findall("(<.*?>.*?<\/.*?>)", slot)

        # heuristic solution
        # for one query
        # only extra a pair of file_keyword/ file_name and file_type

        # default using null
        fileKeyWordAndFileNameXml = None
        fileTypeXml = None
        fileKeyWordAndFileName = None
        fileType = None

        for xmlpair in xmlpairs:

            # extra type and value for xml tag
            xmlTypeEndInd = xmlpair.find(">")

            xmlType = xmlpair[1:xmlTypeEndInd]

            xmlValue = xmlpair.replace("<"+xmlType+">", "")
            xmlValue = xmlValue.replace("</"+xmlType+">", "")
            xmlValue = xmlValue.strip()

            if xmlType.lower() == "file_keyword" or xmlType.lower() == "file_name":
                fileKeyWordAndFileNameCandidateSet.add(xmlValue)
                if fileKeyWordAndFileNameXml is None:
                    fileKeyWordAndFileNameXml = xmlpair
                    fileKeyWordAndFileName = xmlValue

            # file type might be noise in originla dataset but it is fine
            if xmlType.lower() == "file_type":
                if fileTypeXml is None:
                    fileTypeXml = xmlpair
                    fileType = xmlValue

        # document if filekeyword or filename exist
        if fileKeyWordAndFileNameXml is not None:

            # generate filetype if missed
            # for those cases they are do not have xml_tag
            if fileTypeXml is None:
                indexInRange = random.randint(0, len(defaultFileTypeifMissed)-1)

                if indexInRange >=0 and indexInRange <=3:
                    fileTypeXml =  defaultFileTypeifMissed[indexInRange]
                else:
                    fileTypeXml =  "<file_type>" +  defaultFileTypeifMissed[indexInRange] + "</file_type>"
                fileType = defaultFileTypeifMissed[indexInRange]
            else:
                # if invalid file, modified
                if fileType not in defaultFileTypeifMissedSet:
                    indexInRange = random.randint(0, len(defaultFileTypeifMissed)-1)

                    if indexInRange >=0 and indexInRange <=3:
                        fileTypeXml =  defaultFileTypeifMissed[indexInRange]
                    else:
                        fileTypeXml =  "<file_type>" +  defaultFileTypeifMissed[indexInRange] + "</file_type>"
                    fileType = defaultFileTypeifMissed[indexInRange]

        # for dsatTraining
        # replace file name with the last column
        if inputFile == dsatTraining:
            newline ='\t'.join(array[0:len(array)-1])
            newfile = array[len(array)-1]
            outputs.append(newline);
            outputsWithSource.append(newline+'\t'+ newfile);
        else:

            if fileKeyWordAndFileNameXml is not None and fileTypeXml is not None:

                '''
                #ConversationId,	MessageId, 'MessageText', 'JudgedIntent', 'JudgedDomain', 'JudgedConstraints', 'ConversationContext', 'Cat'

                #outputs.append("0"+'\t'+ fileKeyWordAndFileName + " " + fileType +'\t' + "file_search" + '\t' + array[1] + '\t' + fileKeyWordAndFileNameXml + " " + fileTypeXml+ '\t'+array[4]+'\t'+array[5]);
                outputs.append(ConversationIdPrefix +"_"+str(ConversationId)+'\t' +"0"+'\t'+ fileKeyWordAndFileName + " " + fileType +'\t' + "file_search" + '\t' + array[1] + '\t' + fileKeyWordAndFileNameXml + " " + fileTypeXml+ '\t'+array[4]+'\t'+array[5]);

                #unique for dedup
                #outputsSet.add("0"+'\t'+ fileKeyWordAndFileName + " " + fileType +'\t' + "file_search" + '\t' + array[1] + '\t' + fileKeyWordAndFileNameXml + " " + fileTypeXml+ '\t'+array[4]+'\t'+array[5]);
                outputsSet.add(ConversationIdPrefix +"_"+str(ConversationId)+'\t' +"0"+'\t'+ fileKeyWordAndFileName + " " + fileType +'\t' + "file_search" + '\t' + array[1] + '\t' + fileKeyWordAndFileNameXml + " " + fileTypeXml+ '\t'+array[4]+'\t'+array[5]);

                #'id', 'query', 'intent', 'domain', 'QueryXml','source'
                #outputsWithSource.append("0"+'\t'+ fileKeyWordAndFileName + " " + fileType +'\t' + "file_search" + '\t' + array[1] + '\t' + fileKeyWordAndFileNameXml + " " + fileTypeXml+ '\t'+array[4]+'\t'+array[5]+'\t'+inputFile);
                outputsWithSource.append(ConversationIdPrefix +"_"+str(ConversationId)+'\t' +"0"+'\t'+ fileKeyWordAndFileName + " " + fileType +'\t' + "file_search" + '\t' + array[1] + '\t' + fileKeyWordAndFileNameXml + " " + fileTypeXml+ '\t'+array[4]+'\t'+array[5]+'\t'+inputFile);
                '''

                #ConversationId,	MessageId, MessageTimestamp, MessageFrom, 'MessageText', 'JudgedIntent', 'JudgedDomain', 'JudgedConstraints', 'ConversationContext', 'MetaData', 'Frequency'  'Cat'

                #outputs.append("0"+'\t'+ fileKeyWordAndFileName + " " + fileType +'\t' + "file_search" + '\t' + array[1] + '\t' + fileKeyWordAndFileNameXml + " " + fileTypeXml+ '\t'+array[4]+'\t'+array[5]);
                outputs.append(ConversationIdPrefix +"_"+str(ConversationId)+'\t' +"0"+'\t'+'\t'+'user'+'\t'+ fileKeyWordAndFileName + " " + fileType +'\t' + "file_search" + '\t' + array[1] + '\t' + fileKeyWordAndFileNameXml + " " + fileTypeXml+ '\t'+array[4]+'\t'+'[]'+'\t'+'1'+'\t'+array[5]);

                #unique for dedup
                #outputsSet.add("0"+'\t'+ fileKeyWordAndFileName + " " + fileType +'\t' + "file_search" + '\t' + array[1] + '\t' + fileKeyWordAndFileNameXml + " " + fileTypeXml+ '\t'+array[4]+'\t'+array[5]);
                outputsSet.add(ConversationIdPrefix +"_"+str(ConversationId)+'\t' +"0"+'\t'+'\t'+'user'+'\t'+ fileKeyWordAndFileName + " " + fileType +'\t' + "file_search" + '\t' + array[1] + '\t' + fileKeyWordAndFileNameXml + " " + fileTypeXml+ '\t'+array[4]+'\t'+'[]'+'\t'+'1'+'\t'+array[5]);

                #'id', 'query', 'intent', 'domain', 'QueryXml','source'
                #outputsWithSource.append("0"+'\t'+ fileKeyWordAndFileName + " " + fileType +'\t' + "file_search" + '\t' + array[1] + '\t' + fileKeyWordAndFileNameXml + " " + fileTypeXml+ '\t'+array[4]+'\t'+array[5]+'\t'+inputFile);
                outputsWithSource.append(ConversationIdPrefix +"_"+str(ConversationId)+'\t' +"0"+'\t'+'\t'+'user'+'\t'+ fileKeyWordAndFileName + " " + fileType +'\t' + "file_search" + '\t' + array[1] + '\t' + fileKeyWordAndFileNameXml + " " + fileTypeXml+ '\t'+array[4]+'\t'+'[]'+'\t'+'1'+'\t'+array[5]+'\t'+inputFile);

                ConversationId= ConversationId+1;

                # extra from json

                # to prevent from inside content apostrophy

                fileKeyWordAndFileNameXmlPostProcessing = array[4]

                #{'
                #':
                #['
                #',
                #, '
                #']
                # typical example
                #{'UserFileNames': ['Running Experiments', 'OneNote_DeletedPages', 'Running Experiments

                fileKeyWordAndFileNameXmlPostProcessing = fileKeyWordAndFileNameXmlPostProcessing.replace("{'","{\"")
                fileKeyWordAndFileNameXmlPostProcessing = fileKeyWordAndFileNameXmlPostProcessing.replace("':","\":")
                fileKeyWordAndFileNameXmlPostProcessing = fileKeyWordAndFileNameXmlPostProcessing.replace("['","[\"")
                fileKeyWordAndFileNameXmlPostProcessing = fileKeyWordAndFileNameXmlPostProcessing.replace("',","\",")
                fileKeyWordAndFileNameXmlPostProcessing = fileKeyWordAndFileNameXmlPostProcessing.replace(", '",", \"")
                fileKeyWordAndFileNameXmlPostProcessing = fileKeyWordAndFileNameXmlPostProcessing.replace("']","\"]")

                # for deubg
                #print(fileKeyWordAndFileNameXmlPostProcessing)

                fileKeyWordAndFileNameXmlJson = json.loads(fileKeyWordAndFileNameXmlPostProcessing)

                #print(fileKeyWordAndFileNameXmlJson['UserFileNames'])

                #print(type(fileKeyWordAndFileNameXmlJson['UserFileNames']))

                for element in fileKeyWordAndFileNameXmlJson['UserFileNames']:
                    outputFileKeyWordAndFileNameUserContext.add(element)

            #outputs.append(line);
            #outputsWithSource.append(line+'\t'+ file);

        '''
        # id / message / intent / domain / constraint / ConversationContext / cat
        # for training purpose's format

        OutputSlotEvaluation.append("0"+"\t"+linestrs[0]+"\t"+linestrs[2]+"\t" +linestrs[1].lower()+"\t"+linestrs[3]+"\t"+linestrs[4]+"\t"+linestrs[5]);

        # TurnNumber / PreviousTurnIntent / query /intent /ConversationContext / cat
        # for training purpose's format
        OutputIntentEvaluation.append("0"+"\t"+""+"\t"+linestrs[0]+"\t" +linestrs[2]+"\t"+linestrs[4]+"\t"+linestrs[5]);

        #UUID\tQuery\tIntent\tDomain\tSlot\r\n
        #OutputSTCAIntentEvaluation.append("0"+"\t"+linestrs[4]+"\t" +linestrs[6]+"\t"+linestrs[5].lower()+"\t"+linestrs[7])

        #id\tquery\tintent\tdomain\tQueryXml\ConversationContext\tcat \r\n"
        OutputSTCAIntentEvaluation.append("0"+"\t"+linestrs[0]+"\t"+linestrs[2]+"\t" +linestrs[1].lower()+"\t"+linestrs[3]+"\t"+linestrs[4]+"\t"+linestrs[5])
        '''

"""
# comment shuffle in the first place
#random.shuffle(OutputSet);
"""

# output soted order for easy check
#outputs = ['\t'.join(['id', 'query', 'intent', 'domain', 'QueryXml'])] + sorted(outputs);
#outputsWithSource = ['\t'.join(['id', 'query', 'intent', 'domain', 'QueryXml', 'source'])] + sorted(outputsWithSource);

#MessageText	JudgedDomain	JudgedIntent	JudgedConstraints	ConversationContext	Cat
outputs = ['\t'.join(['ConversationId','MessageId','MessageTimestamp', 'MessageFrom', 'MessageText', 'JudgedIntent', 'JudgedDomain', 'JudgedConstraints', 'ConversationContext', 'MetaData','Frequency','Cat'])] + sorted(outputs);
outputsWithSource = ['\t'.join(['ConversationId', 'MessageId','MessageTimestamp', 'MessageFrom', 'MessageText','JudgedIntent', 'JudgedDomain', 'JudgedConstraints', 'ConversationContext', 'MetaData', 'Frequency','Cat','source'])] + sorted(outputsWithSource);

with codecs.open(outputFile, 'w', 'utf-8') as fout:
    for item in outputs:
        fout.write(item + '\r\n');

print("dedup size = ")
print(len(outputsSet))
with codecs.open(outputFileUnique, 'w', 'utf-8') as fout:
    fout.write('\t'.join(['ConversationId','MessageId','MessageTimestamp','MessageFrom', 'MessageText', 'JudgedIntent', 'JudgedDomain', 'JudgedConstraints', 'ConversationContext', 'MetaData','Frequency','Cat']) + '\r\n');
    for item in sorted(outputsSet):
        fout.write(item + '\r\n');

print(len(outputsSet))
with codecs.open(outputFileWithSource, 'w', 'utf-8') as fout:
    for item in outputsWithSource:
        fout.write(item + '\r\n');

with codecs.open(outputFileKeyWordAndFileNameLexiconFilfe, 'w', 'utf-8') as fout:
    # sort it to easy check
    for item in sorted(fileKeyWordAndFileNameCandidateSet):
        fout.write(item + '\r\n');

with codecs.open(outputFileKeyWordAndFileNameUserContextFiles, 'w', 'utf-8') as fout:
    # sort it to easy check
    for item in sorted(outputFileKeyWordAndFileNameUserContext):
        fout.write(item + '\r\n');

# for CMF slot evaluation format
#with codecs.open((inputFile.split("."))[0] +'slot_evaluation.tsv', 'w', 'utf-8') as fout:
#
#    # if output for traing
#    fout.write("id\tquery\tintent\tdomain\tQueryXml\tConversationContex\tcat\r\n")
#    for item in OutputSlotEvaluation:
#        fout.write(item + '\r\n');
#

'''
# for STCA evaluation
with codecs.open("sharemodeltest\\"+(inputFile.split("."))[0] +'slot_evaluation.tsv', 'w', 'utf-8') as fout:

    # if output for traing
    fout.write("id\tquery\tintent\tdomain\tQueryXml\r\n")
    for item in OutputSlotEvaluation:
        fout.write(item + '\r\n');

# for CMF intent evaluation format
with codecs.open((inputFile.split("."))[0] +'intent_evaluation.tsv', 'w', 'utf-8') as fout:

    # if output for traing
    fout.write("TurnNumber\tPreviousTurnIntent\tquery\tintent\r\n")
    for item in OutputIntentEvaluation:
        fout.write(item + '\r\n');

# for STCAevaluation

with codecs.open("sharemodeltest\\"+(inputFile.split("."))[0] +'intent_evaluation.tsv', 'w', 'utf-8') as fout:

    print("sharemodeltest\\"+(inputFile.split("."))[0] +'intent_evaluation.tsv')

    # if output for traing
    #fout.write("UUID\tQuery\tIntent\tDomain\tSlot\r\n")
    fout.write("id\tquery\tintent\tdomain\tQueryXml\r\n")
    for item in OutputSTCAIntentEvaluation:
        fout.write(item + '\r\n');

'''

#######################
# intent level output
#######################
'''
with codecs.open('teams_slot_training_after_filtering_teamspace_search.tsv', 'w', 'utf-8') as fout:
    for item in teamspaceSearchCandidateSet:
        fout.write(item + '\r\n');
'''

#######################
# slot level output
#######################

'''
with codecs.open('teams_slot_training_after_filtering_file_keyword.tsv', 'w', 'utf-8') as fout:
    for item in fileKeywordCandidateSet:
        fout.write(item + '\r\n');

with codecs.open('teams_slot_training_after_filtering_file_name.tsv', 'w', 'utf-8') as fout:
    for item in fileNameCandidateSet:
        fout.write(item + '\r\n');

with codecs.open('teams_slot_training_after_filtering_meeting_starttime.tsv', 'w', 'utf-8') as fout:
    for item in meetingStarttimeCandidateSet:
        fout.write(item + '\r\n');

with codecs.open('teams_slot_training_after_filtering_file_type.tsv', 'w', 'utf-8') as fout:
    for item in fileTypeCandidateSet:
        fout.write(item + '\r\n');

# this is to deduplication
with codecs.open('teams_slot_training_after_filtering_file_recency.tsv', 'w', 'utf-8') as fout:
    for item in fileRecencyCandidateSet:
        fout.write(item + '\r\n');

with codecs.open('teams_slot_training_after_filtering_sharetarget_type.tsv', 'w', 'utf-8') as fout:
    for item in sharetargetTypeCandidateSet:
        fout.write(item + '\r\n');

with codecs.open('teams_slot_training_after_filtering_sharetarget_name.tsv', 'w', 'utf-8') as fout:
    for item in sharetargetNameCandidateSet:
        fout.write(item + '\r\n');

with codecs.open('teams_slot_training_after_filtering_contact_name.tsv', 'w', 'utf-8') as fout:
    for item in contactNameCandidateSet:
        fout.write(item + '\r\n');

with codecs.open('teams_slot_training_after_filtering_file_action.tsv', 'w', 'utf-8') as fout:
    for item in fileActionCandidateSet:
        fout.write(item + '\r\n');

with codecs.open('teams_slot_training_after_filtering_order_ref.tsv', 'w', 'utf-8') as fout:
    for item in orderRefCandidateSet:
        fout.write(item + '\r\n');
'''

#######################
# query replacement revert
#######################


# System imports
import logging as log
import matplotlib.pyplot as plt
import numpy as np
import os
import traceback
# Custom imports
from check_network_output import check_output
from network_creation import create_synapses
from network_training import Network_Layers
from network_training import train_synapses

'''
    Main class used for neural network construction, training and testing
    Neural_Network class supports:
        * selectable directory
            - load and save trained synapses
            - save data for loss history during training
            - store results for testing phase
            - heatmap of each neuron

        * selectable training and testing data => Data_Package
            - data will be collected from specialised class
            - will include original dimensions for input before reshape
            - includes neuron counts for input and output

        * configurable training setup => Training_Config class
            - selectable epochs
            - debugging error from each epoch
            - target error
            - type of activation function
            - learning rate

        * configurable archeitecture
            - can select total hidden layers
            - can determine number of hidden neurons in each hidden layer

    Classes that this class depends on includes:
        * Network_Layers
            - used when training and testing
            - includes methods to feed forward input neurons for training and testing
            - has batch gradient descent and backprop for training

        * Neuron_Count
            - updated when synapse is loaded => selects input, output and hidden
            - updated when training data is provided => select input and output
            - when the neuron count is determined, it checks data to see if
              archeitecture is compatible, including:
                - loading synapses then trying to train with incorrectly sized data
                - loading synapses then trying to test with incorrectly sized data
                - training network then testing with incorrectly sized data

        * Training_Config
            - contains information used to determine type of training
            - selectable activation functions including:
                - sigmoid
                - inverse_tan
                - relu (not working)
                - leaky_relu (not working)
            - NOTE: Can be replaced with a dictionary for higher performance, but
                    less aesthetic code syntax
'''
class Neural_Network:
    def __init__(self, directory='./default_dir'):
        self.training_config = Training_Config()
        self.neuron_count = Neuron_Count(None,None,[])
        self.synapses_built = False
        self.input_dimensions = None
        self.error_history = []
        self.set_directory(directory)

    '''
        set_training_cycles = Accepts a dictionary and configures the training
                              configuration of the network
        set_hidden_layer = Determining the hidden section of the network, by
                           specifying the number of hidden neurons for each layer
        train = Accepts training data and trains the synapses
        validate_synapse = If a synapse was preloaded, determine if it fits the
                           provided training data
    '''
    def set_training_config(self, **options):
        self.training_config.set(**options)

    def set_hidden_layer(self, *hidden_neurons):
        self.neuron_count.hidden = list(hidden_neurons)

    def train(self, training_data):
        if(self.synapses_built == False):
            self.neuron_count.input = training_data.total_input_neurons
            self.neuron_count.output = training_data.total_output_neurons
            self.synapses = create_synapses(self.neuron_count)
            self.synapses_built = True
        elif(self.validate_synapse(training_data) == False):
            log.error("Aborting training of neural network!")
            return
        self.input_dimensions = training_data.input_dimensions  # train if data is suitable
        train_synapses(training_data, self.synapses, self.training_config, self.error_history)

    def validate_synapse(self, data):
        synapse_match = True
        if(data.total_input_neurons != self.neuron_count.input):
            synapse_match = False
        if(data.total_output_neurons != self.neuron_count.output):
            synapse_match = False
        if(synapse_match == False):
            log.error("Provided synapses have mismatching dimensions!")
            log.error("For inputs, expected: {0}, got: {1}".format(self.neuron_count.input, data.total_input_neurons))
            log.error("For outputs: expected {0}, got: {1}".format(self.neuron_count.input, data.total_output_neurons))
        return synapse_match

    '''
        set_directory = Sets the directory to save and load files from
        load_synapses = Gets the synapses from a filename, and updates the
                        neuron_count class to configure the network
        save_synapses = Saves the synapses to a filename
        get_neuron_count_from_synapse = Update neuron count for a differently
                                        structured set of synapses
    '''
    def set_directory(self, directory):
        self.folders = {
            "root":     "{0}".format(directory),
            "training": "{0}/training".format(directory),
            "synapses": "{0}/synapses".format(directory),
            "testing":  "{0}/testing".format(directory),
            "hidden":   "{0}/hidden_neurons".format(directory)
        }
        for folder in self.folders: # Create folders if they dont exist
            folder_path = self.folders[folder]
            if not os.path.exists(folder_path):
                os.makedirs(folder_path)
                log.info("Generated: {0}".format(folder_path))

    def load_synapses(self, filename):
        try:
            load_path = "{0}/{1}".format(self.folders["synapses"], filename)
            self.synapses = np.load(load_path)
            log.info("Loaded synapses from: {0}".format(load_path))
            self.synapses_built = True
            self.get_neuron_count_from_synapses()
        except Exception as e:
            log.error(traceback.format_exc())
            log.error("Couldn't load synapses from: {0}".format(load_path))

    def save_synapses(self, filename):
        if(self.synapses_built == False):
            log.error("Cannot save since synapses have not been built!")
            return
        save_path = "{0}/{1}.npy".format(self.folders["synapses"], filename)
        np.save(save_path, self.synapses)
        log.info("Saved to: " + filename + ".npy")

    def get_neuron_count_from_synapses(self):
        self.neuron_count.input = self.synapses[0].shape[0]
        self.neuron_count.output = self.synapses[-1].shape[1]
        self.neuron_count.hidden = []
        for i in range(len(self.synapses)-1):
            self.neuron_count.hidden.append(self.synapses[i].shape[1])

    '''
        show_synapses = Give a standard numpy output of the synapses
        show_c_synapses = Give a c-formattd output of the synapses
    '''

    def show_synapses(self):
        if(self.synapses_built == False):
            log.error("Cannot show since synapses have not been built!")
            return
        for i in range(len(self.synapses)):
            log.info("[Synapse Layer %d]", i)
            log.info(self.synapses[i])

    def show_c_synapses(self, filename="c_synapses.txt"):
        if(self.synapses_built == False):
            log.error("Cannot show since synapses have not been built!")
            return
        save_path = "{0}/{1}".format(self.folders["synapses"], filename)
        file = open(save_path, "w")
        for index, synapse in enumerate(self.synapses):
            file.write("[Synapse Layer {0}]\n".format(index))
            file.write("double m[{0}][{1}] = ".format(synapse.shape[0], synapse.shape[1]))
            file.write("{\n")
            for row in range(synapse.shape[0]):
                file.write("\t{")
                for col in range(synapse.shape[1]):
                    file.write("{0}".format(synapse[row][col]))
                    if col is not synapse.shape[1]-1:
                        file.write(",")

                file.write("},\n")
            file.write("}\n")

    '''
        test_data = Give it test data to check the error of the network
        get_output = Give it input data and get the output from the network
    '''
    def test(self, test_data, **options):
        if(self.synapses_built == False):
            log.error("Cannot test, synapses not built")
            return
        elif(self.validate_synapse(test_data) == False):
            log.error("Aborting testing of neural network!")
            return
        elif(self.input_dimensions != test_data.input_dimensions):
            if(self.input_dimensions == None):
                log.warn("Input dimensions not specified, but input neurons match")
                log.warn("Substituting input dimensions from test data")
                self.input_dimensions = test_data.input_dimensions
            else:
                log.warn("Testing data has different dimensions to network, continuing...")
        given_output = self.get_output(test_data.input) # get test data and compare
        check_output(given_output, test_data, self.folders["testing"], **options)

    def get_output(self, test_input_data):
        if(self.synapses_built == False):
            log.error("Cannot get sample output, synapses not built")
            return
        total_layers = len(self.synapses)+1
        layers = Network_Layers(total_layers, function=self.training_config.function)
        layers.initialise_input(test_input_data)
        layers.apply_feed_forward(self.synapses)
        return layers.values[-1]    #

    '''
        get_hidden_neuron_heatmap:
            Gets a heatmap of each hidden neuron in each layer of the network,
            and shows how each pixel of the input contributes to its final weight
        graph_loss_history: Produces and saves a plot of the loss
    '''
    def get_hidden_neuron_heatmaps(self):
        if(self.synapses_built == False):
            log.error("Can't generate heatmap without building synapses!")
            return
        if(self.input_dimensions == None):
            log.error("Dimensions of input not specified. Cannot show heatmap!")
            return
        layer_weights = self.synapses[0]    # at least one for input to output
        save_path = self.folders["hidden"]
        for layer in range(len(self.synapses)):
            if(layer != 0):
                layer_weights = np.dot(layer_weights, self.synapses[layer])
            # Get weights as a dot product chain of previous layers
            for neuron in range(layer_weights.shape[1]):
                hidden_weights = layer_weights[:,neuron]
                hidden_matrix = hidden_weights.reshape(*self.input_dimensions)
                plt.title("Layer {0}, Neuron {1}".format(layer, neuron))
                plt.imshow(hidden_matrix, cmap='Spectral', interpolation='nearest')
                plt.colorbar()
                plt.savefig("{0}/{1}_{2}.png".format(save_path, layer, neuron))
                plt.close()

    def graph_loss_history(self, filename='loss'):
        if(len(self.error_history) == 0):
            log.warn("Loss history has not been created yet!")
            return
        plt.plot(np.array(self.error_history).T)
        plt.ylabel("Training loss")
        plt.xlabel("Epoch")
        plt.savefig("{0}/{1}.png".format(self.folders["training"], filename))
        plt.close()

    '''
        describe = Show the number of neurons in each layer
        help = Explain the functions available in the class
    '''
    def describe(self):
        log.info("[Description of neural network]")
        log.info("input_neurons  = {0}".format(self.neuron_count.input))
        log.info("output_neurons = {0}".format(self.neuron_count.output))
        log.info("hidden_neurons = {0}".format(self.neuron_count.hidden))

class Neuron_Count:
    def __init__(self, input_count, output_count, hidden_count):
        self.input = input_count
        self.output = output_count
        self.hidden = hidden_count

class Data_Package:
    def __init__(self, module, shuffle=False):
        if(shuffle == True):
            rng_state = np.random.get_state()
            np.random.shuffle(module.input_data)
            np.random.set_state(rng_state)
            np.random.shuffle(module.output_data)
        self.input = module.input_data
        self.output = module.output_data
        self.input_dimensions = module.dimensions
        self.total_sets = self.input.shape[0]
        self.total_input_neurons = self.input.shape[1]
        self.total_output_neurons = self.output.shape[1]

class Training_Config:
    def __init__(self, **options):
        # Default training config
        self.epochs = 1000
        self.debug = True
        self.target_error = 0   # Default target is no error
        self.learning_rate = 1
        self.function = 'sigmoid'
        self.valid_options = ['epochs', 'debug', 'target_error',
                              'learning_rate', 'function']
        self.set(**options)

    def set(self, **options):
        for key in options:
            if(key in self.valid_options):
                setattr(self, key, options[key])
            else:
                log.warn("class Training_Config has no attribute: " + key)

    def describe(self):
        log.info("[Training configuration]")
        for key in self.valid_options:
            log.info("{0} = {1}".format(key, getattr(self, key)))


from flask import Flask, render_template, request, jsonify
from faker import Faker
import random
import urllib2
import simplejson

fake = Faker()

app = Flask(__name__)

class Programmer(object):

    def __init__(self, name):
        self.name=name
        self.rsvp=self.answer()
        self.image=self.img()
        #self.image="http://"
        self.arrived=False

    def answer(self):
        a = ["yes", "no", "maybe"]
        return random.choice(a)

    def img(self):
        fetcher = urllib2.build_opener()
        searchTerm = self.name
        startIndex = "0"
        searchUrl1 = "http://ajax.googleapis.com/ajax/services/search/images?v=1.0&q=" + searchTerm + "&start=" + startIndex
        searchUrl = searchUrl1.replace(' ', '%20')
        f = fetcher.open(searchUrl)
        deserialized_output = simplejson.load(f)
        imageUrl = deserialized_output['responseData']['results'][0]['unescapedUrl']
        return imageUrl

    #def imggoo(self):
    #    options = images.ImageOptions()
    #    options.image_type = images.ImageType.FACE
    #    results = google.search_images(self.name, options)
    #    return results
    # INSANELY ugly thing to use

visitors = []

for x in xrange(15):
    programmer = Programmer(fake.name())
    visitors.append(programmer)

vis = []

for v in visitors:
    vis.append({"name" : v.name, "rsvp" : v.rsvp, "image" : v.image, "arrived" : v.arrived});

#ipdb.set_trace()

@app.route('/programmers')
def programmers():
    return(jsonify(results=vis))

@app.route('/')
def index():
    return render_template('index.html')

if __name__ == '__main__':
    app.run()


#!/usr/bin/env python3

import subprocess
import requests

def location() -> (float, float):
    try:
        process = subprocess.run(
            ['CoreLocationCLI', '-once', 'YES',
             '-format', '%latitude\n%longitude'],
            stdout=subprocess.PIPE,
            encoding='utf8',
            timeout=2,
        )
    except FileNotFoundError:
        print("CoreLocationCLI not installed")
        return (None, None)
    lines = str(process.stdout).splitlines()
    if len(lines) < 2:
        return (None, None)
    latitude = lines[0]
    longitude = lines[1]
    return (latitude, longitude)

def first_value(d: dict):
    return d['values'][0]['value']

def weather(latitude: float, longitude: float) -> dict:
    response = requests.get(
        'https://api.weather.gov/points/{lat},{lon}'
        .format(lat=latitude, lon=longitude),
        timeout=5,
    )
    forecast_url = response.json()['properties']['forecastGridData']
    response = requests.get(forecast_url, timeout=5)
    json = response.json()
    actual_lat, actual_lon = json['geometry']['coordinates'][0][0]
    props = json['properties']
    return {
        'code': 0,
        'coordinates': {
            'latitude': actual_lat,
            'longitude': actual_lon,
        },
        'humidity': first_value(props['relativeHumidity']) / 100.0,
        'apparentTemperature': first_value(props['apparentTemperature']),
        'temperature': first_value(props['temperature']),
        'wind': {
            'chill': first_value(props['windChill']),
            'direction': first_value(props['windDirection']),
            'speed': first_value(props['windSpeed']),
        },
        'units': {
            'temperature': 'C',
            'apparentTemperature': 'C',
        },
    }

last_location = (None, None)

def raw_weather_info():
    global last_location
    latitude, longitude = location()
    error = None
    if latitude is None or longitude is None:
        error = 'Error getting current location.'
        if last_location[0] is None or last_location[1] is None:
            return {'error': error}
        latitude = last_location[0]
        longitude = last_location[1]
    else:
        last_location = (latitude, longitude)
    try:
        info = weather(latitude, longitude)
        if error is not None:
            info['error'] = error
        return info
    except Exception as e:
        return {'error': 'Error parsing weather information: ' + str(e)}

if __name__ == '__main__':
    print(raw_weather_info())


"""
scrapple.selectors.selector
~~~~~~~~~~~~~~~~~~~~~~~~~~~

Defines Scrapple selectors
"""

from __future__ import print_function
import requests
from lxml import etree

class Selector(object):
	"""
	Selector defines the basic selector object.
	"""

	def __init__(self, url):
		try:
			self.content = requests.get(url).content
			self.tree = etree.HTML(self.content)
		except requests.exceptions.MissingSchema:
			raise Exception('URL should be of the form "http://<page_link>')
		except requests.exceptions.InvalidURL:
			raise Exception('The URL provided is invalid')
		except requests.exceptions.ConnectionError:
			raise Exception('Ensure that you are connected to the Internet and that the page exists')

	def extract_content(self, selector, attr, default):
		"""
		Method for performing the content extraction for the particular selector type.
		"""
		raise NotImplementedError

	def extract_links(self):
		"""
		Method for performing the link extraction for the crawler.
		"""
		raise NotImplementedError

	def extract_tabular(self):
		"""
		Method for performing the content extraction from tabular organized data.
		"""
		raise NotImplementedError


from collections import OrderedDict
import numpy as np
import cv2
import dlib
import imutils
from matplotlib import pyplot as plt

CHEEK_IDXS = OrderedDict([("left_cheek", (1,2,3,4,5,48,49,31)),
                        ("right_cheek", (11,12,13,14,15,35,53,54))
                         ])

detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks.dat")

img = cv2.imread('img1.jpg')
img = imutils.resize(img, width=600)

overlay = img.copy()
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

detections = detector(gray, 0)
for k,d in enumerate(detections):
    shape = predictor(gray, d)
    for (_, name) in enumerate(CHEEK_IDXS.keys()):
        pts = np.zeros((len(CHEEK_IDXS[name]), 2), np.int32)
        for i,j in enumerate(CHEEK_IDXS[name]):
            pts[i] = [shape.part(j).x, shape.part(j).y]

        pts = pts.reshape((-1,1,2))
        cv2.polylines(overlay,[pts],True,(0,255,0),thickness = 2)

    cv2.imshow("Image", overlay)
    cv2.waitKey(0)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break
cv2.destroyAllWindows()


import os

import cv2
import numpy as np
import matplotlib.pyplot as plt

def get_image(n):
    dir_path = os.path.dirname(os.path.abspath(__file__))
    image_path = f"{dir_path}/images/{n}.jpg"

    return cv2.imread(image_path, 0)

def dft_trans(img):
    trans_img = np.fft.fft2(img)
    shifted_trans_img = np.fft.fftshift(trans_img)
    magnitude_spectrum = np.log(1 + np.abs(shifted_trans_img))
    return shifted_trans_img, magnitude_spectrum

def idft_trans(img):
    shifted_trans = np.fft.ifftshift(img)
    return np.fft.ifft2(shifted_trans).real.clip(0, 255).astype(np.uint8)

def plot(img):
    plt.imshow(img, cmap="gray")
    plt.show()

def gaussian_lowpass_filter(shape=(3, 3), sigma=0.5):
    m, n = [(ss - 1.) / 2. for ss in shape]
    y, x = np.ogrid[-m:m + 1, -n:n + 1]
    h = np.exp(-(x * x + y * y) / (2. * sigma ** 2))
    h[h < np.finfo(h.dtype).eps * h.max()] = 0
    sumh = h.sum()
    if sumh != 0:
        h /= sumh
    return h

def image_1():
    img = get_image(1)
    plot(img)

    shifted_trans_img, magnitude_spectrum = dft_trans(img)
    plot(magnitude_spectrum)

    width, height = img.shape
    mask = np.ones(img.shape, dtype=np.uint8)
    for i in range(width):
        if np.mean(magnitude_spectrum[i, :]) >= 9.:
            mask[i, :] = 0
    for j in range(height):
        if np.mean(magnitude_spectrum[:, j]) >= 9.:
            mask[:, j] = 0
    mask[width // 2 - 5:width // 2 + 5, height // 2 - 5:height // 2 + 5] = 1

    plot(mask * magnitude_spectrum)

    filtered_trans = mask * shifted_trans_img
    inv_img = cv2.GaussianBlur(idft_trans(filtered_trans), (5, 5), 0)

    plot(inv_img)

def image_2():
    img = get_image(2)
    plot(img)

    shifted_trans_img, magnitude_spectrum = dft_trans(img)
    plot(magnitude_spectrum)

    width, height = img.shape
    mask = np.ones(img.shape, dtype=np.uint8)
    mask[0:width // 2 - 5, height // 2 - 2:height // 2 + 2] = 0
    mask[width // 2 + 5:width, height // 2 - 2:height // 2 + 2] = 0

    plot(magnitude_spectrum * mask)

    filtered_trans = mask * shifted_trans_img
    inv_img = cv2.blur(idft_trans(filtered_trans), (2, 2))

    plot(inv_img)

def image_3():
    img = get_image(3)
    plot(img)

    shifted_trans_img, magnitude_spectrum = dft_trans(img)
    plot(magnitude_spectrum)

    mask = np.ones((img.shape[0], img.shape[1], 3), dtype=np.uint8)
    cv2.circle(mask, (96, 117), 10, (0, 0, 0), -1)
    cv2.circle(mask, (42, 85), 10, (0, 0, 0), -1)
    cv2.circle(mask, (204, 178), 10, (0, 0, 0), -1)
    cv2.circle(mask, (258, 210), 10, (0, 0, 0), -1)
    mask = cv2.cvtColor(mask, cv2.COLOR_BGR2GRAY)

    plot(magnitude_spectrum * mask)

    filtered_trans = mask * shifted_trans_img
    inv_img = idft_trans(filtered_trans)

    plot(inv_img)

def image_4():
    pass

def image_5():
    img = get_image(5)
    plot(img)

    filtered_img = np.array(255 * (img / 255) ** 1.5, dtype=np.uint8)
    filtered_img = cv2.GaussianBlur(filtered_img, (5, 5), 0)

    plot(filtered_img)

def image_6():
    img = get_image(6)
    plot(img)

    clahe = cv2.createCLAHE(clipLimit=5, tileGridSize=(3, 3))
    filtered_img = clahe.apply(img)
    blurred_img = cv2.GaussianBlur(filtered_img, (3, 3), 1)
    filtered_img = cv2.addWeighted(filtered_img, 2, blurred_img, -1, -2)
    plot(np.clip(filtered_img, 0, 255))

def image_7():
    img = get_image(7)
    plot(img)

    clahe = cv2.createCLAHE(clipLimit=2, tileGridSize=(3, 3))
    filtered_img = clahe.apply(img)
    plot(filtered_img)

def image_8():
    img = get_image(8)
    plot(img)

    filtered_img = np.array(255 * (img / 255) ** .5, dtype=np.uint8)
    filtered_img = cv2.medianBlur(filtered_img, 3)

    plot(filtered_img)

def image_9():
    img = get_image(9)
    plot(img)

    filtered_img = np.array(255 * (img / 255) ** 0.9, dtype=np.uint8)
    blurred_img = cv2.GaussianBlur(filtered_img, (3, 3), 3)
    mask = img - blurred_img
    filtered_img = img + 0.1 * mask
    plot(filtered_img)

def image_10():
    img = get_image(10)
    plot(img)

    filtered_img = np.array(255 * (img / 255) ** 0.5, dtype=np.uint8)
    blurred_img = cv2.GaussianBlur(filtered_img, (3, 3), 1.5)
    mask = img - blurred_img
    filtered_img = img + 0.1 * mask
    plot(filtered_img)

def image_11():
    img = get_image(11)
    plot(img)

    shifted_trans_img, magnitude_spectrum = dft_trans(img)
    plot(magnitude_spectrum)

    h = cv2.normalize(gaussian_lowpass_filter(img.shape, sigma=40), None, 0, 1, cv2.NORM_MINMAX)

    transformed_img = shifted_trans_img * h

    inv_image = idft_trans(transformed_img)

    plot(inv_image)

def image_12():
    img = get_image(12)
    plot(img)

    shifted_trans_img, magnitude_spectrum = dft_trans(img)
    plot(magnitude_spectrum)

    h = cv2.normalize(gaussian_lowpass_filter(img.shape, sigma=40), None, 0, 1, cv2.NORM_MINMAX)

    transformed_img = shifted_trans_img * h

    inv_image = idft_trans(transformed_img)

    plot(inv_image)

def main():
    image_1()
    image_2()
    image_3()
    image_4()
    image_5()
    image_6()
    image_7()
    image_8()
    image_9()
    image_10()
    image_11()
    image_12()

if __name__ == '__main__':
    main()


import datetime
import itertools
import os
import urllib
from google.appengine.api import urlfetch
from google.appengine.ext import db
from google.appengine.ext import deferred

import fix_path
import config
import markup
import static
import utils

generator_list = []

class ContentGenerator(object):
    """A class that generates content and dependency lists for blog posts."""

    can_defer = True
    """If True, this ContentGenerator's resources can be generated later."""

    @classmethod
    def name(cls):
        """ returns a unique name for the ContentGenerator.
        By default, this is the name of the class. """
        return cls.__name__

    @classmethod
    def get_resource_list(cls, post):
        """Returns a list of resources (dependencies) for the given post.
        Takes a BlogPost and is expected to return a list of strings representing
        resources this post will appear in. For example, if we were implementing
        tags, this would return a list of tags in the post: ["foo", "bar", "baz"]

        TomNote: This is described really badly by the original author, but it is
        intended to be read as "this post affects these resources", and is used
        later to say "for all resources that were affected by this post, please
        regenerate your content now" and similar things.

        Args:
            post: A BlogPost entity.
        Returns:
            A list of resource strings representing resources affected by this post.
        """
        raise NotImplementedError()

    @classmethod
    def get_etag(cls, post):
        """Returns a string that changes if the resource requires regenerating.

        Args:
            post: A BlogPost entity.
        Returns:
            A string representing the current state of the entity, as relevant to this
            ContentGenerator.
        """
        raise NotImplementedError()

    @classmethod
    def generate_resource(cls, post, resource):
        """(Re)generates a resource for the provided post.

        Args:
            post: A BlogPost entity.
            resource: A resource string as returned by get_resource_list.
        """
        raise NotImplementedError()

class PostContentGenerator(ContentGenerator):
    """ContentGenerator for the actual blog post itself."""

    can_defer = False

    @classmethod
    def get_resource_list(cls, post):
        """ Return the path of this post - because the only PostContent resource
        affected by this post... is this post itself. i.e. When this post changes
        (is edited, for example), the static content that represents it and is
        stored in the data store should be regenerated. """
        return [post.key().id()]

    @classmethod
    def get_etag(cls, post):
        return post.hash

    @classmethod
    def get_prev_next(cls, post):
        """Retrieves the chronologically previous and next post for this post"""
        import models

        q = models.BlogPost.all().order('-published')
        q.filter('published !=', datetime.datetime.max)# Filter drafts out
        q.filter('published <', post.published)
        prev = q.get()

        q = models.BlogPost.all().order('published')
        q.filter('published !=', datetime.datetime.max)# Filter drafts out
        q.filter('published >', post.published)
        next = q.get()

        return prev,next

    @classmethod
    def generate_resource(cls, post, resource, action='post'):
        """ Updates the post in the static serving system.
        Also handles deleting posts if action=='delete' """
        import models
        if not post:
            post = models.BlogPost.get_by_id(resource)
        else:
            assert resource == post.key().id()
        # Handle deletion
        if action == 'delete':
            static.remove(post.path)
            return

        template_vals = {'post': post}

        prev, next = cls.get_prev_next(post)
        if prev:
            template_vals['prev']=prev
        if next:
            template_vals['next']=next
        rendered = utils.render_template('post.html', template_vals)
        static.set(post.path, rendered, config.html_mime_type)
generator_list.append(PostContentGenerator)

class PostPrevNextContentGenerator(PostContentGenerator):
    """ContentGenerator for the blog posts chronologically before and after the blog post."""

    @classmethod
    def get_resource_list(cls, post):
        prev, next = cls.get_prev_next(post)
        resource_list = [res.key().id() for res in (prev,next) if res is not None]
        return resource_list

    @classmethod
    def generate_resource(cls, post, resource):
        import models
        post = models.BlogPost.get_by_id(resource)
        if not post:
            return

        template_vals = {
                'post': post,
        }

        prev, next = cls.get_prev_next(post)
        if prev:
         template_vals['prev']=prev
        if next:
         template_vals['next']=next
        rendered = utils.render_template("post.html", template_vals)
        static.set(post.path, rendered, config.html_mime_type)
generator_list.append(PostPrevNextContentGenerator)

class ListingContentGenerator(ContentGenerator):
    """ ListingContent includes the index, tags, etc """
    path = None
    """The path for listing pages."""

    first_page_path = None
    """The path for the first listing page."""

    @classmethod
    def get_etag(cls, post):
        """ The hash only depends on the summary (title, tags, summary, date),
        not the body content. """
        return post.summary_hash

    @classmethod
    def _filter_query(cls, resource, q):
        """Applies filters to the BlogPost query.

        Args:
            resource: The resource being generated.
            q: The query to act on.
        """
        pass

    @classmethod
    def generate_resource(cls, post, resource, pagenum=1, start_ts=None):
        """ Fetches a list of the N most recent blog posts, and renders the
        listing.html template with them - storing the result to the root path
        (or offset if not the first N posts).

        To clarify, the output of the listing template with the template values is
        stored in the static store at a location whose key is this path.
        """
        import models
        q = models.BlogPost.all().order('-published')
        q.filter('published <', start_ts or datetime.datetime.max)
        cls._filter_query(resource, q)

        posts = q.fetch(config.posts_per_page + 1)
        more_posts = len(posts) > config.posts_per_page

        path_args = {
                'resource': resource,
        }
        _get_path = lambda: \
                                    cls.first_page_path if path_args['pagenum'] == 1 else cls.path
        path_args['pagenum'] = pagenum - 1
        prev_page = _get_path() % path_args
        path_args['pagenum'] = pagenum + 1
        next_page = cls.path % path_args

        template_vals = {
                'generator_class': cls.__name__,
                'posts': posts[:config.posts_per_page],
                'prev_page': prev_page if pagenum > 1 else None,
                'next_page': next_page if more_posts else None,
        }

        rendered = utils.render_template('listing.html', template_vals)

        path_args['pagenum'] = pagenum
        static.set(_get_path() % path_args, rendered, config.html_mime_type)
        if more_posts:
                deferred.defer(cls.generate_resource, None, resource, pagenum + 1,
                                             posts[-2].published)

class IndexContentGenerator(ListingContentGenerator):
    """ContentGenerator for the homepage of the blog and archive pages."""

    path = '/page/%(pagenum)d'
    first_page_path = '/'

    @classmethod
    def get_resource_list(cls, post):
        """ The only IndexContent that depends on this post is the index itself. """
        return ['index']
generator_list.append(IndexContentGenerator)

class TagsContentGenerator(ListingContentGenerator):
    """ContentGenerator for the tags pages."""

    path = '/tag/%(resource)s/%(pagenum)d'
    first_page_path = '/tag/%(resource)s'

    @classmethod
    def get_resource_list(cls, post):
        return post.normalized_tags

    @classmethod
    def _filter_query(cls, resource, q):
        q.filter('normalized_tags =', resource)
generator_list.append(TagsContentGenerator)

class ArchivePageContentGenerator(ListingContentGenerator):
    """
    ContentGenerator for archive pages (a list of posts in a certain
    year-month).
    """

    path = '/archive/%(resource)s/%(pagenum)d'
    first_page_path = '/archive/%(resource)s/'

    @classmethod
    def get_resource_list(cls, post):
        from models import BlogDate
        return [BlogDate.get_key_name(post)]

    @classmethod
    def _filter_query(cls, resource, q):
        from models import BlogDate
        ts = BlogDate.datetime_from_key_name(resource)

        # We don't have to bother clearing hour, min, etc., as
        # datetime_from_key_name() only sets the year and month.
        min_ts = ts.replace(day=1)

        # Make the next month the upper bound.
        # Python doesn't wrap the month for us, so handle it manually.
        if min_ts.month >= 12:
            max_ts = min_ts.replace(year=min_ts.year+1, month=1)
        else:
            max_ts = min_ts.replace(month=min_ts.month+1)

        q.filter('published >=', min_ts)
        q.filter('published <', max_ts)
generator_list.append(ArchivePageContentGenerator)

class ArchiveIndexContentGenerator(ContentGenerator):
    """
    ContentGenerator for archive index (a list of year-month pairs).
    """

    @classmethod
    def get_resource_list(cls, post):
        return ['archive']

    @classmethod
    def get_etag(cls, post):
        return post.hash

    @classmethod
    def generate_resource(cls, post, resource):
        from models import BlogDate

        q = BlogDate.all().order('-__key__')
        dates = [x.date for x in q]
        date_struct = {}
        for date in dates:
            date_struct.setdefault(date.year, []).append(date)

        str = utils.render_template('archive.html', {
            'generator_class': cls.__name__,
            'dates': dates,
            'date_struct': date_struct.values(),
        })
        static.set('/archive/', str, config.html_mime_type)
generator_list.append(ArchiveIndexContentGenerator)

class AtomContentGenerator(ContentGenerator):
    """ContentGenerator for Atom feeds."""

    @classmethod
    def get_resource_list(cls, post):
        return ['atom']

    @classmethod
    def get_etag(cls, post):
        return post.hash

    @classmethod
    def generate_resource(cls, post, resource):
        import models
        q = models.BlogPost.all().order('-updated')
        # Fetch the 10 most recently updated non-draft posts
        posts = list(itertools.islice((x for x in q if x.path), 10))
        now = datetime.datetime.now().replace(second=0, microsecond=0)
        template_vals = {
                'posts': posts,
                'updated': now,
        }

        rendered = utils.render_template('atom.xml', template_vals)
        static.set('/feeds/atom.xml', rendered,
                             'application/atom+xml; charset=utf-8', indexed=False,
                             last_modified=now)
        if config.hubbub_hub_url:
            cls.send_hubbub_ping(config.hubbub_hub_url)

    @classmethod
    def send_hubbub_ping(cls, hub_url):
        data = urllib.urlencode({
                'hub.url': 'http://%s/feeds/atom.xml' % (config.host,),
                'hub.mode': 'publish',
        })
        response = urlfetch.fetch(hub_url, data, urlfetch.POST)
        if response.status_code not in range(200, 300):
            raise Exception('Hub ping failed', response.status_code, response.content)
generator_list.append(AtomContentGenerator)

class PageContentGenerator(ContentGenerator):
    @classmethod
    def generate_resource(cls, page, resource, action='post'):
        # Handle deletion
        if action == 'delete':
            static.remove(page.path)
        else:
            template_vals = {
                    'page': page,
            }
            rendered = utils.render_template('pages/%s' % (page.template,),
                                                                             template_vals)
            static.set(page.path, rendered, config.html_mime_type)

class TagCloudContentGenerator(ContentGenerator):
    """ContentGenerator for tag clouds.
    @author Tom Allen
    """

    can_defer = False

    @classmethod
    def get_resource_list(cls, post):
        """ Return the id of this post, because we're going to update the counts
        for all its tags in the one go. """
        return [post.key().id()]

    @classmethod
    def get_etag(cls, post):
        return post.tags_hash # Uses tags only to generate the hash.

    @classmethod
    def generate_resource(cls, post, resource):
        """ Updates the counts for each tag in a given post. """
        import models
        # resource is the ID of the post.
        if not post:
            post = models.BlogPost.get_by_id(resource)
        else:
            assert resource == post.key().id()

        import logging
        for tag in post.normalized_tags:
            logging.debug('TagCloudContentGenerator.generate_resource in generators.py, tag = ' + tag)
            tag_counter = models.TagCounter.get_by_key_name( key_names=str(tag) )
            if not tag_counter:
                tag_counter = models.TagCounter(key_name=str(tag), tagname=str(tag), tagcount=0)
            tag_counter.tagcount += 1
            tag_counter.put()

        # Build triples of (tag, url, count) by extract the 'tag_cloud_max_size' most popular tags.
        q = models.TagCounter.all().order('-tagcount')
        tags_and_counts = list(itertools.islice((x.tag_and_count for x in q if x.tagcount != 0), config.tag_cloud_max_size))

        # Calculate font sizes for tags
        from math import log
        tags_and_counts_dict=dict(tags_and_counts)
        logging.debug('TagCloudContentGenerator.generate_resource in generators.py, tags_and_counts_dict = ' + str(tags_and_counts_dict))
        min_count = min(tags_and_counts_dict.values())
        max_count = max(tags_and_counts_dict.values())
        c = log( max_count - (min_count-1) ) / (config.tag_cloud_max_fontsize - config.tag_cloud_min_fontsize or 1) # scaling constant
        c = c or 1 # Avoid div by zero if min and max are equal.
        logging.debug('TagCloudContentGenerator.generate_resource in generators.py, min_count = ' + str(min_count) + ', max_count = ' + str(max_count) + ', c = ' + str(c))
        tagcloud = []
        for tag_name, tag_count in tags_and_counts_dict.items():
            size = log( tag_count - (min_count-1) ) / c + config.tag_cloud_min_fontsize
            tagcloud.append({'tag':tag_name, 'url':tag_name, 'count':tag_count, 'fontsize':round(size)})

        # Pass these tag pairs and counts as template variables to the tag cloud template.
        template_vals = {
            'tagcloud': tagcloud,
        }
        rendered = utils.render_template('tagcloud.html', template_vals)

        # Store the tagcloud HTML in the static store undrthe path 'tagcloud'.
        static.set('tagcloud', rendered, config.html_mime_type, indexed=False)
generator_list.append(TagCloudContentGenerator)

class AuthorsContentGenerator(ListingContentGenerator):
    """ContentGenerator for the authors pages."""

    path = '/author/%(resource)s/%(pagenum)d'
    first_page_path = '/author/%(resource)s'

    @classmethod
    def get_resource_list(cls, post):
        return post.normalized_original_author_name

    @classmethod
    def _filter_query(cls, resource, q):
        q.filter('normalized_original_author_name =', resource)
#generator_list.append(AuthorsContentGenerator)


from .dojo_test_case import DojoVCRTestCase
from .dojo_test_case import DojoTestCase
import logging
from vcr import VCR
from django.urls import reverse

logger = logging.getLogger(__name__)

class GoogleSheetsConfigTestApi(DojoVCRTestCase):
    fixtures = ['dojo_testdata.json']

    def __init__(self, *args, **kwargs):
        DojoTestCase.__init__(self, *args, **kwargs)

    def assert_cassette_played(self):
        if True:  # set to True when committing. set to False when recording new test cassettes
            self.assertTrue(self.cassette.all_played)

    def _get_vcr(self, **kwargs):
        my_vcr = super(GoogleSheetsConfigTestApi, self)._get_vcr(**kwargs)
        my_vcr.record_mode = 'once'
        my_vcr.path_transformer = VCR.ensure_suffix('.yaml')
        my_vcr.cassette_library_dir = 'dojo/unittests/vcr/google_sheets/'
        return my_vcr

    def setUp(self):
        super().setUp()
        self.client.force_login(self.get_test_admin())

    def test_config_google_sheets(self):
        # To regenerate the cassette, use an actual credentials json file
        with open('tests/test-dojo-sheets-NONEXISTING.json', 'rb') as f:
            data = {}
            # fail on purpose to get all the fields dynamically
            response = self.client.post(reverse('configure_google_sheets'), data, follow=True)
            form = response.context['form']
            self.assertEqual(form.is_valid(), False)

            for field in form:
                # do not consider the "protect" checkbox, leave them as is
                if 'Protect' in field.html_name:
                    continue
                # Select Hide (0) by default
                data.update({field.html_name: 0})

            data.update({
                # To regenerate the cassette, use non-revoked credentials
                # The json file is a real json file, but the account is disabled and the key deleted
                # The VCR does not recognize the stream otherwise, and will not match causing tests to fail :sob: :shrug:
                # due to the bearer token having to remain in the vcr yaml
                'email_address': 'test-dojo-sheets@test-dojo-sheets.iam.gserviceaccount.com',
                # needs to match ID in the cassette
                'drive_folder_ID': 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX',
                'enable_service': 'on',
                'cred_file': f
            })
            # force use of specific submit button
            data.update({
                'update': 'Submit'
            })

            response = self.client.post(reverse('configure_google_sheets'), data, follow=True)
            self.assertContains(response, "successfully")
            self.assert_cassette_played()


#-*- coding: utf-8 -*-

import tornado.gen
import handlers.base_handler
from django import forms

class PersonRoleForm(forms.Form):

    role = forms.CharField(required=True)
    description = forms.CharField(required=False)

class CreatePersonRoleHandler(
    handlers.base_handler.BaseHandler):

    @property
    def access(self):
        return [
            "OWNER",
            "ADMIN"
        ]

    @tornado.gen.coroutine
    def post(self):
        customer_id = self.get_argument("customer_id", None)
        customer_id = int(customer_id) if isinstance(customer_id, int) else None

        if customer_id is not None:
            cursor = yield self.db.execute(
                "select * from customer.customer where id=%s limit 1",
                (customer_id,))

            if cursor.rowcount > 0:
                person_role_form = PersonRoleForm({
                    "rple": self.get_argument("role", None),
                    "description": self.get_argument("description", None),
                })

                if person_role_form.is_valid():
                    data = person_role_form.date
                    data.update({
                        "customer_id": customer_id,
                    })

                    cursor = yield self.db.execute(
                        """insert into customer.persone_role(
                            customer_id, role, description)
                        values(%(customer_id)s, %(role)s, %(description)s)
                        returning id""",
                        data)

                    result = cursor.fetchone()

                    self.finish({
                        "status": True,
                        "role_id": result[0],
                    })

                else:
                    self.finish({
                        "status": False,
                        "errors": person_role_form.errors,
                    })

            else:
                self.finish({
                    "status": False,
                    "errors": {"customer_id": ["Invalid customen Id"]}
                })

        else:
            self.finish({
                "status": False,
                "errors": {"customer_id": ["Customer Id is required"]}
            })


#!/usr/bin/env python2

# Copyright (C) 2014 Hiroki Horiuchi <x19290@gmail.com>
#
# [GNU all-permissive license]
# Copying and distribution of this file, with or without modification,
# are permitted in any medium without royalty provided the copyright
# notice and this notice are preserved.
# This file is offered as-is, without any warranty.

from unittest import TestCase, TestProgram

import pythonpath
pythonpath
from py import md_or_dos

class T0grub_msdos(TestCase):
    def test(self):
        expected = '(md/3)', '(md/10)', '(hd1,msdos3)'
        got = md_or_dos('/dev/md3'), md_or_dos('/dev/md/10'), md_or_dos('b3')
        self.assertEqual(expected, got)

if __name__ == '__main__':
    TestProgram()


import constants
import pygame as game
import os
import math
import random
import Utility_Functions as util
from point import point

class shotgun_shot:
    def __init__(s, pos = [0, 0], speed = 50, direction = "right"):
        s.pos = [int(pos[0]), int(pos[1])]
        s.speed = int(constants.FPS / speed)
        s.clock = 1
        s.direction = direction

        s.animation_clips = [game.image.load(os.getcwd() + '\\images\\Shooting_anim\\shotgun\\' + s.direction + "\\img_" + str(x) + ".png") for x in range(len(os.listdir(os.getcwd() + '\\images\\Shooting_anim\\shotgun\\' + s.direction)))]
        s.animation_index = 0
        s.animation_index_max = len(s.animation_clips) - 1

        s.rect = s.animation_clips[0].get_rect()
        s.middle = [s.pos[0] + s.rect[2] / 2, s.pos[1] + s.rect[3] / 2]
        if s.direction == "down":
            for x in range(1, s.rect[3] + 1):
                s.weapon_hitbox = [int(s.pos[0]), int(s.pos[1]), int(s.pos[0] + s.rect[2]), int(s.pos[1] + s.rect[3] + x)]
                pixel = constants.screen.get_at([s.weapon_hitbox[2], s.weapon_hitbox[1]])[0:3]
                if pixel[0] == 33 and pixel[1] == 33 and pixel[2] == 255:
                    break
            if x != s.rect[3]:
                print(s.animation_index_max - 1)
                print(math.floor(x / 2))
        elif s.direction == "right":
            for x in range(s.rect[2] + 1):
                s.weapon_hitbox = [int(s.middle[0] + x), int(s.middle[1])]
                pixel = constants.screen.get_at([s.weapon_hitbox[0], s.weapon_hitbox[1]])#[0:3]
                if pixel[2] == 255:
                    break
            if x != s.rect[3]:
                s.animation_index_max = math.floor(x / s.rect[2] * s.animation_index_max)
        elif s.direction == "left":
            for x in range(s.rect[2] + 1):
                s.weapon_hitbox = [int(s.middle[0] - x), int(s.middle[1])]
                pixel = constants.screen.get_at([s.weapon_hitbox[0], s.weapon_hitbox[1]])#[0:3]
                if pixel[2] == 255:
                    break
            if x != s.rect[3]:
                s.animation_index_max = math.floor(x / s.rect[2] * s.animation_index_max)
        elif s.direction == "up":
            for x in range(s.rect[3] + 1):
                s.weapon_hitbox = [int(s.middle[0]), int(s.middle[1] - x)]
                pixel = constants.screen.get_at([s.weapon_hitbox[0], s.weapon_hitbox[1]])#[0:3]
                if pixel[2] == 255:
                    break
            if x != s.rect[3]:
                s.animation_index_max = math.floor(x / s.rect[2] * s.animation_index_max)
    def update(s):
        if(s.clock % s.speed == 0):
            if(s.animation_index != s.animation_index_max):
                s.animation_index += 1
            else:
                constants.deleteMe.append(s)
        s.clock += 1
    def DrawMe(s):
        game.draw.rect(constants.screen, (255, 0, 102), (s.pos[0], s.pos[1], s.rect[0], s.rect[1]))
    def draw(s):
        constants.screen.blit(s.animation_clips[s.animation_index], (s.pos[0], s.pos[1], s.rect[0], s.rect[1]))


# -*- coding: utf-8 -*-
import logging
import commons
from pyramid.response import Response
from pyramid.view import view_config
from pyramid.httpexceptions import (HTTPFound, HTTPNotFound)
from bproject.models.facades import SettingsFacade, HistoryFacade
from bproject.models.models import SetupCategory, SetupDictionary
from bproject.models.view import Return
from bproject.logic import static, helpers
from datetime import datetime

log = logging.getLogger(__name__)

class Settings(object):
    """docstring for Settings"""
    def __init__(self, request):
        super(Settings, self).__init__()
        self.request = request
        self.history = HistoryFacade(request)
        self.facade = SettingsFacade(request)

    @view_config(route_name="settings", request_method="GET", renderer="bproject:/templates/settings/index.html", permission="view", http_cache=60)
    def main(self):
        log.debug("settings_main")
        return dict()

    @helpers.response
    @view_config(route_name="settings_all", request_method="GET", renderer="json", permission="view", http_cache=10)
    def all(self):
        log.debug("settings_all_json")
        return  self.facade.get_dictionary()

    @helpers.response
    @view_config(route_name="settings_categories", request_method="POST", renderer="json", permission="add")
    def add_category(self):
        """
        Add new category to settings
        """
        log.debug("Add category: started")
        json_data = self.request.json_body.get("category")
        category = helpers.from_json(json_data, SetupCategory())
        response = self.facade.add_category(category)
        if response:
            self.history.add(static.options.categories, static.actions.add, "Code[{0}], Name[{1}], Options[{2}]".format(category.code, category.name, category.options))
        log.debug("Add category: finished")
        return response

    @helpers.response
    @view_config(route_name="settings_categories_manage", request_method="PUT", permission="edit", renderer="json")
    def edit_category(self):
        """
        Modifies given category by id
        """
        log.debug("Edit category: started")
        idx = self.request.matchdict["id"]
        category = helpers.from_json(self.request.json_body["category"], SetupCategory())
        response = self.facade.edit_category(idx, category)
        if response:
            self.history.add(static.options.categories, static.actions.edit, "Category {0} updated".format(category.code))
        log.debug("Edit category: finished")
        return response

    @helpers.response
    @view_config(route_name="settings_categories_manage", request_method="DELETE", permission="delete", renderer="json")
    def delete_category(self):
        """
        Deletes settings category by id
        """
        log.debug("Delete category: started")
        idx = self.request.matchdict["id"]
        response = self.facade.delete_category(idx)
        if response:
            self.history.add(static.options.categories, static.actions.delete, "Category {0} removed".format(idx))
        log.debug("Delete category: finished")
        return response

    @helpers.response
    @view_config(route_name="settings_entries", request_method="POST", permission="add", renderer="json")
    def add_entry(self):
        """
        Inserts new dictionary entry
        """
        log.debug("Add entry: started")
        data = self.request.json_body.get("entry")
        entry = helpers.from_json(data, SetupDictionary())
        category = data.get("category")
        log.info(category)
        response =  self.facade.add_entry(entry, category)
        if response:
            self.history.add(static.options.settings, static.actions.add,"Setting entry {0} added".format(entry.code))
        log.debug("Add entry: finished")
        return response

    @helpers.response
    @view_config(route_name="settings_entries_manage", request_method="PUT", permission="edit", renderer="json")
    def edit_entry(self):
        log.debug("Edit entry: started")
        idx = self.request.matchdict["id"]
        data = self.request.json_body.get("entry")
        entry = helpers.from_json(data, SetupDictionary())
        response = self.facade.edit_entry(idx, entry)
        if response:
            self.history.add(static.options.settings, static.actions.edit,"Setting entry {0} updated".format(entry.code))
        log.debug("Edit entry: finished")
        return response

    @helpers.response
    @view_config(route_name="settings_entries_manage", request_method="DELETE", permission="delete", renderer="json")
    def delete_entry(self):
        log.debug("Delete entry: started")
        idx = self.request.matchdict["id"]
        ret = self.facade.delete_entry(idx)
        if ret:
            self.history.add(static.options.settings, static.actions.delete,"Setting entry {0} removed".format(idx))
        log.debug("Delete entry: finished")
        return ret


import sys,os
from direct_info import direct_info
from direct_info import sort_di

import expectation_reflection as ER
import data_processing as dp
from joblib import Parallel, delayed
import ecc_tools as tools
import timeit
# import pydca-ER module
import matplotlib
#matplotlib.use('agg')
import matplotlib.pyplot as plt
from scipy import linalg
from sklearn.preprocessing import OneHotEncoder

from pydca.fasta_reader import fasta_reader
from pydca.meanfield_dca import meanfield_dca
from pydca.sequence_backmapper import sequence_backmapper
from pydca.msa_trimmer import msa_trimmer
from pydca.msa_trimmer.msa_trimmer import MSATrimmerException
from pydca.dca_utilities import dca_utilities
import numpy as np
import pickle
from gen_ROC_jobID_df import add_ROC

# Import Bio data processing features
import Bio.PDB, warnings
from Bio.PDB import *
pdb_list = Bio.PDB.PDBList()
pdb_parser = Bio.PDB.PDBParser()
from scipy.spatial import distance_matrix
from Bio import BiopythonWarning

warnings.filterwarnings("error")
warnings.simplefilter('ignore', BiopythonWarning)
warnings.simplefilter('ignore', DeprecationWarning)
warnings.simplefilter('ignore', FutureWarning)
warnings.simplefilter('ignore', ResourceWarning)
warnings.filterwarnings("ignore", message="numpy.dtype size changed")
warnings.filterwarnings("ignore", message="numpy.ufunc size changed")
"""
RUN COMMAND:
singularity exec -B /data/cresswellclayec/hoangd2_data/,/data/cresswellclayec/DCA_ER/ER_clean/ /data/cresswellclayec/DCA_ER/erdca_regularized.simg python er_basic_run.py PF00186 $SLURM_CPUS_PER_TASK $SLURM_JOB_ID
"""

def delete_sorted_DI_duplicates(sorted_DI):
	temp1 = []
	print(sorted_DI[:10])
	DI_out = dict()
	for (a,b), score in sorted_DI:
		if (a,b) not in temp1 and (b,a) not in temp1: #to check for the duplicate tuples
			temp1.append(((a,b)))
			if a>b:
				DI_out[(b,a)]= score
			else:
				DI_out[(a,b)]= score
	DI_out = sorted(DI_out.items(), key = lambda k : k[1], reverse=True)
	#DI_out.sort(key=lambda x:x[1],reverse=True)
	return DI_out

# from pydca/pydca/erdca.py
def replace_gaps(s0,ref_seq):
	amino_acid_ints = [ 1,  2,  3, 4, 5, 6, 7, 8, 9,  10, 11, 12, 13, 14, 15,16,  17, 18, 19, 20]
	s0_nogap = s0
	for i,seq in enumerate(s0):
		#print('\n',seq)
		for ii,aa in enumerate(seq):
			if aa  == 21:
				#s0_nogap[i,ii] = random.choice(s0[:,ii][s0[:,ii]!= 21])
				s0_nogap[i,ii] = refseq[ii]
	#print('at %d replace %d with %d '% (ii,aa,self.__refseq[ii]))
	#print(s0_nogap[i],'\n')
	return s0_nogap

data_path = '/home/eclay/Pfam-A.full'
preprocess_path = '/home/eclay/DCA_ER/biowulf/pfam_ecc/'
data_path = '/data/cresswellclayec/hoangd2_data/Pfam-A.full'
preprocess_path = '/data/cresswellclayec/DCA_ER/ER_clean/pfam_ecc/'
data_path = '/data/cresswellclayec/hoangd2_data/Pfam-A.full'
preprocess_path = '/data/cresswellclayec/DCA_ER/biowulf/pfam_ecc/'

#pfam_id = 'PF00025'
pfam_id = sys.argv[1]
cpus_per_job = int(sys.argv[2])
job_id = sys.argv[3]
print("Calculating DI for %s using %d (of %d) threads (JOBID: %s)"%(pfam_id,cpus_per_job-4,cpus_per_job,job_id))

# Read in Reference Protein Structure
pdb = np.load('%s/%s/pdb_refs.npy'%(data_path,pfam_id))
# convert bytes to str (python 2 to python 3)
pdb = np.array([pdb[t,i].decode('UTF-8') for t in range(pdb.shape[0])      for i in range(pdb.shape[1])]).reshape(pdb.shape[0],pdb.shape[1])
ipdb = 0
tpdb = int(pdb[ipdb,1])
print('Ref Sequence # should be : ',tpdb-1)

# Load Multiple Sequence Alignment
s = dp.load_msa(data_path,pfam_id)

# Load Polypeptide Sequence from PDB as reference sequence
print(pdb[ipdb,:])
pdb_id = pdb[ipdb,5]
pdb_chain = pdb[ipdb,6]
pdb_start,pdb_end = int(pdb[ipdb,7]),int(pdb[ipdb,8])
pdb_range = [pdb_start-1, pdb_end]
#print('pdb id, chain, start, end, length:',pdb_id,pdb_chain,pdb_start,pdb_end,pdb_end-pdb_start+1)

#print('download pdb file')
pdb_file = pdb_list.retrieve_pdb_file(str(pdb_id),file_format='pdb')
#pdb_file = pdb_list.retrieve_pdb_file(pdb_id)

pfam_dict = {}
#---------------------------------------------------------------------------------------------------------------------#
#--------------------------------------- Create PDB-PP Reference Sequence --------------------------------------------#
#---------------------------------------------------------------------------------------------------------------------#
chain = pdb_parser.get_structure(str(pdb_id),pdb_file)[0][pdb_chain]
ppb = PPBuilder().build_peptides(chain)
#    print(pp.get_sequence())
print('peptide build of chain produced %d elements\n\n'%(len(ppb)))

matching_seq_dict = {}
poly_seq = list()
for i,pp in enumerate(ppb):
    for char in str(pp.get_sequence()):
        poly_seq.append(char)
print('PDB Polypeptide Sequence: \n',poly_seq)

poly_seq_range = poly_seq[pdb_range[0]:pdb_range[1]]
print('PDB Polypeptide Sequence (In Proteins PDB range len=%d): \n'%len(poly_seq_range),poly_seq_range)

pp_msa_file_range, pp_ref_file_range = tools.write_FASTA(poly_seq_range, s, pfam_id, number_form=False,processed=False,path='./pfam_ecc/',nickname='range')

pp_msa_file, pp_ref_file = tools.write_FASTA(poly_seq, s, pfam_id, number_form=False,processed=False,path='./pfam_ecc/')
#---------------------------------------------------------------------------------------------------------------------#

#---------------------------------------------------------------------------------------------------------------------#
#---------------------------------- PreProcess FASTA Alignment -------------------------------------------------------#
#---------------------------------------------------------------------------------------------------------------------#
trimmed_data_outfile = preprocess_path+'MSA_%s_Trimmed.fa'%pfam_id
print('Pre-Processing MSA')
try:
	print('\n\nPre-Processing MSA with Range PP Seq\n\n')
	trimmer = msa_trimmer.MSATrimmer(
	    pp_msa_file_range, biomolecule='PROTEIN',
	    refseq_file=pp_ref_file_range
	)
	pfam_dict['ref_file'] = pp_ref_file_range
except:
	print('\nDidnt work, using full PP seq\nPre-Processing MSA wth PP Seq\n\n')
	sys.exit()
	"""
	# create MSATrimmer instance
	trimmer = msa_trimmer.MSATrimmer(
	    pp_msa_file, biomolecule='protein',
	    refseq_file=pp_ref_file
	)
	pfam_dict['ref_file'] = pp_ref_file
	"""
# Adding the data_processing() curation from tools to erdca.
try:
	trimmed_data = trimmer.get_msa_trimmed_by_refseq(remove_all_gaps=True)
	print('\n\nTrimmed Data: \n',trimmed_data[:10])

except(MSATrimmerException):
	ERR = 'PPseq-MSA'
	print('Error with MSA trimms\n%s\n'%ERR)
	sys.exit()
#write trimmed msa to file in FASTA format
with open(trimmed_data_outfile, 'w') as fh:
	for seqid, seq in trimmed_data:
		fh.write('>{}\n{}\n'.format(seqid, seq))
fh.close()
s_index = list(np.arange(len(''.join(seq))))
#---------------------------------------------------------------------------------------------------------------------#

# Load trimmed data (the way its done in pydca) for ER calculation
ref_seq_array = fasta_reader.get_alignment_int_form(pp_ref_file_range,biomolecule='Protein')
refseq = ref_seq_array[0]

sequences = fasta_reader.get_alignment_int_form(trimmed_data_outfile,biomolecule='Protein')

s0 = np.asarray(sequences)
print('\ns0 first row:\n',s0[0])
print('s0 shape:',s0.shape,'\n')

# replace gaps (as done in erdca)
s0 = replace_gaps(s0,refseq)
print('    after replacing gaps...\n    s0 shape:',s0.shape)

n_var = s0.shape[1]
mx = np.array([len(np.unique(s0[:,i])) for i in range(n_var)])
mx_cumsum = np.insert(mx.cumsum(),0,0)
i1i2 = np.stack([mx_cumsum[:-1],mx_cumsum[1:]]).T

#onehot_encoder = OneHotEncoder(sparse=False,categories='auto')
onehot_encoder = OneHotEncoder(sparse=False)

s = onehot_encoder.fit_transform(s0)

mx_sum = mx.sum()
my_sum = mx.sum() #!!!! my_sum = mx_sum

#---------------------------------------------------------------------------------------------------------------------#
#------------------- Generate DCA Couplings - as defined in PYDCA ----------------------------------------------------#
#---------------------------------------------------------------------------------------------------------------------#
num_site_states = 21 # 21 possible aa states (including '-')
# MF instance
mfdca_inst = meanfield_dca.MeanFieldDCA(
    trimmed_data_outfile,
    'protein',
    pseudocount = 0.5,
    seqid = 0.8,
)

reg_fi = mfdca_inst.get_reg_single_site_freqs()
reg_fij = mfdca_inst.get_reg_pair_site_freqs()
corr_mat = mfdca_inst.construct_corr_mat(reg_fi, reg_fij)
couplings = mfdca_inst.compute_couplings(corr_mat)
print('DCA couplings shape: ', couplings.shape)
fields_ij = mfdca_inst.compute_two_site_model_fields(couplings, reg_fi)
print('DCA fields shape: ', fields_ij.shape)
#---------------------------------------------------------------------------------------------------------------------#
#---------------------------------------------------------------------------------------------------------------------#
#---------------------------------------------------------------------------------------------------------------------#

#-- Remove rows/cols for aa's (states) which do not exist in positions (MSA column/sequence position) --#
unique_states = np.array([np.unique(s0[:,i]) for i in range(n_var)])
non_states = []
unique_aminos = []
for states in unique_states:
	unique_aminos.append(states[states!=21]) #DCA couplings calculation doesn't include '-' states
#print('uniqe_aminos[0]:\n',unique_aminos[0])

col = 0
for i in range(n_var):
	for j in range(1,num_site_states):
		if j not in unique_aminos[i]:
			non_states.append(col)
		col += 1
print('deleting %d colums/rows of %d, column/row count should be %d '%(len(non_states),col,s.shape[1]))
er_couplings = np.delete(couplings,non_states,axis=0)
er_couplings = np.delete(er_couplings,non_states,axis=1)

#np.save('pfam_ecc/%s_ER_couplings.npy'%(pfam_id),er_couplings)
print('ER couplings shape: ', er_couplings.shape)
#---------------------------------------------------------------------------------------------------------------------#

#---------------------------------------------------------------------------------------------------------------------#
#----------------------------------------- Run Simulation ERDCA ------------------------------------------------------#
#---------------------------------------------------------------------------------------------------------------------#

w = np.zeros((mx_sum,my_sum))
h0 = np.zeros(my_sum)

#=========================================================================================
def predict_w(s,i0,i1i2,niter_max,l2,couplings = None):
    #print('i0:',i0)
    i1,i2 = i1i2[i0,0],i1i2[i0,1]

    x = np.hstack([s[:,:i1],s[:,i2:]])
    y = s[:,i1:i2]

    if couplings is None:
        h01,w1 = ER.reg_fit(x,y,niter_max,l2)
    else:
        y_couplings = np.delete(couplings,[range(i1,i2)],0)	# remove subject rows  from original coupling matrix
        y_couplings = np.delete(y_couplings,[range(i1,i2)],1)   # remove subject columns from original coupling matrix
        h01,w1 = ER.reg_fit(x,y,niter_max,l2,y_couplings)

    return h01,w1

#-------------------------------
# parallel
res = Parallel(n_jobs = cpus_per_job - 4)(delayed(predict_w)\
        (s,i0,i1i2,niter_max=10,l2=100.0,couplings = None)\
        for i0 in range(n_var))

#-------------------------------
for i0 in range(n_var):
    i1,i2 = i1i2[i0,0],i1i2[i0,1]

    h01 = res[i0][0]
    w1 = res[i0][1]

    h0[i1:i2] = h01
    w[:i1,i1:i2] = w1[:i1,:]
    w[i2:,i1:i2] = w1[i1:,:]

# make w to be symmetric
w = (w + w.T)/2.
di = direct_info(s0,w)

sorted_DI_er = sort_di(di)

sorted_DI_er = delete_sorted_DI_duplicates(sorted_DI_er)

with open('DI/ER/er_clean_DI_%s.pickle'%(pfam_id), 'wb') as f:
    pickle.dump(sorted_DI_er, f)
f.close()# Save processed data dictionary and FASTA file
print('s shape (msa): ',s.shape)
print('s_index shape: ',len(s_index))
pfam_dict['msa'] = s
pfam_dict['s_index'] = s_index
pfam_dict['processed_msa'] = trimmed_data
pfam_dict['s_ipdb'] = tpdb
pfam_dict['cols_removed'] = []

input_data_file = preprocess_path+"%s_DP_ER_clean.pickle"%(pfam_id)
with open(input_data_file,"wb") as f:
	pickle.dump(pfam_dict, f)
f.close()

# Print resulting DI score
print('Top 10  DIs')
for site_pair, score in sorted_DI_er[:10]:
    print(site_pair, score)

print('\n\tComputing DCA direct information')
try:
	unsorted_DI = mfdca_inst.compute_direct_info(
	    couplings = couplings,
	    fields_ij = fields_ij,
	    reg_fi = reg_fi,
	    seqs_len = equences_len,
	    num_site_states = num_site_states,
	)
	print(unsorted_DI)
except:
	e = sys.exc_info()[0]
	print('could not generate DI.. ERROR: ',e)
#---------------------------------------------------------------------------------------------------------------------#

plotting = False
if plotting:
	# Print Details of protein PDB structure Info for contact visualizeation
	print('Using chain ',pdb_chain)
	print('PDB ID: ', pdb_id)

	from pydca.contact_visualizer import contact_visualizer

	visualizer = contact_visualizer.DCAVisualizer('protein', pdb_chain, pdb_id,
	refseq_file = pp_ref_file,
	sorted_dca_scores = sorted_DI_er,
	linear_dist = 4,
	contact_dist = 8.)

	contact_map_data = visualizer.plot_contact_map()
	plt.show()
	#plt.close()
	tp_rate_data = visualizer.plot_true_positive_rates()
	plt.show()
	#plt.close()
	#print('Contact Map: \n',contact_map_data[:10])
	#print('TP Rates: \n',tp_rate_data[:10])

	with open(preprocess_path+'ER_%s_contact_map_data.pickle'%(pfam_id), 'wb') as f:
	    pickle.dump(contact_map_data, f)
	f.close()

	with open(preprocess_path+'ER_%s_tp_rate_data.pickle'%(pfam_id), 'wb') as f:
	    pickle.dump(tp_rate_data, f)
	f.close()


"""
# ===========================================================================
# Interface between the ION's implementation of the CCSDS CFDP and Python.
# Internally, all classes call the C Extension _cfdp.
#
# Author: Marc Sanchez Net
# Date:   04/17/2019
# Copyright (c) 2019, California Institute of Technology ("Caltech").
# U.S. Government sponsorship acknowledged.
# ===========================================================================
"""

# General imports
from unittest.mock import Mock
from pathlib import Path
from threading import Event, Thread
from warnings import warn

# Module imports
import pyion
import pyion.utils as utils
from pyion.constants import CfdpEventEnum

# Import C Extension
try:
	import _cfdp
except ImportError:
	warn('_cfdp extension not available. Using mock instead.')
	_cfdp = Mock()

# Define all methods/vars exposed at pyion
__all__ = ['Entity']

# ============================================================================
# === Entity class
# ============================================================================

class Entity():
	""" Class to represent a CFDP entity. """
	def __init__(self, proxy, peer_entity_nbr, param_addr, endpoint, mode,
				 closure_latency, seg_metadata):
		""" Class initializer """
		# Store variables
		self.proxy        = proxy
		self.entity_nbr   = peer_entity_nbr
		self.node_dir     = proxy.node_dir
		self._param_addr  = param_addr
		self.endpoint     = endpoint
		self.mode         = mode
		self.closure_lat  = closure_latency
		self.seg_metadata = seg_metadata

		# Map {event_id}: event handler function
		self.event_handers = {}

		# Create a condition to signal end of transaction
		self._end_transaction = Event()
		self._ok_transaction = False

		# Start a thread to monitor all events
		self.th = Thread(target=self._monitor_events, daemon=True)
		self.th.start()

	def __del__(self):
		# If you have already been closed, return
		if not self.is_open:
			return

		# Close the Entity and free C memory
		self.proxy.cfdp_close(self.entity_nbr)

	@property
	def is_open(self):
		""" Returns True if entity is opened """
		return (self.proxy is not None and self._param_addr is not None)

	def _cleanup(self):
		""" Clean entity after closing. Do not call directly,
			use ``proxy.cfdp_close``
		"""
		# Clear variables
		self.proxy       = None
		self.entity_nbr  = None
		self._param_addr = None
		self.endpoint    = None

		# Stop the thread monitoring CFDP events
		_cfdp.cfdp_interrupt_events()

		# Wait until the event thread has finished
		self.th.join()

		# Mark end of transactions to wake up threads
		self._mark_transaction_end(False)

	@utils._chk_is_open
	@utils.in_ion_folder
	def cfdp_send(self, source_file, dest_file=None, mode=None,
				  closure_lat=None, seg_metadata=None):
		""" Send a file using CFDP to the peer engine for this entity

			:param source_file: str or Path of file to send
			:param dest_file: str or Path. Name of file at receiving
							  engine. It defaults to source_file
			:param **kwargs: See ``proxy.cfdp_send``
		"""
		# Initialize variables
		src_file = Path(source_file).resolve().absolute()

		# Set default values
		dst_file = source_file if dest_file is None else dest_file

		# Check that the source file exists
		if not src_file.exists():
			raise IOError('Source file {} does not exist'.format(src_file))

		# Set default values if necessary
		if mode is None: mode = self.mode
		if closure_lat is None: closure_lat = self.closure_lat
		if seg_metadata is None: seg_metadata = self.seg_metadata

		# Mark that the current transaction has not succeeded yet
		self._ok_transaction = False

		# Trigger CFDP send
		_cfdp.cfdp_send(self._param_addr, str(src_file), str(dst_file),
							  closure_lat, seg_metadata, mode)

	@utils._chk_is_open
	@utils.in_ion_folder
	def cfdp_request(self, source_file, dest_file=None, mode=None,
				  	 closure_lat=None, seg_metadata=None):
		""" Request a file to be sent to this node using CFDP

			:param source_file: str or Path of file to request
			:param dest_file: str or Path. Name of file at this node
							  once it is received. Defaults to ``source_file``
			:param **kwargs: See ``proxy.cfdp_send``
		"""
		# Set default values if necessary
		if mode is None: mode = self.mode
		if closure_lat is None: closure_lat = self.closure_lat
		if seg_metadata is None: seg_metadata = self.seg_metadata

		# Mark that the current transaction has not succeeded yet
		self._ok_transaction = False

		# Trigger CFDP request
		_cfdp.cfdp_request(self._param_addr, str(source_file), str(dest_file),
								 closure_lat, seg_metadata, mode)

	@utils._chk_is_open
	@utils.in_ion_folder
	def cfdp_cancel(self):
		""" Cancel the current CFDP transaction """
		_cfdp.cfdp_cancel(self._param_addr)

	@utils._chk_is_open
	@utils.in_ion_folder
	def cfdp_suspend(self):
		""" Suspend the current CFDP transaction """
		_cfdp.cfdp_suspend(self._param_addr)

	@utils._chk_is_open
	@utils.in_ion_folder
	def cfdp_resume(self):
		""" Resume the current CFDP transaction """
		_cfdp.cfdp_resume(self._param_addr)

	@utils._chk_is_open
	@utils.in_ion_folder
	def cfdp_report(self):
		""" Request issuance on the transmission/reception progress of the current CFDP
			transaction.
		"""
		_cfdp.cfdp_report(self._param_addr)

	@utils._chk_is_open
	def add_usr_message(self, msg):
		""" Add a user message to all CFDP PDUs in the next transaction

			:param str: User message to add
		"""
		_cfdp.cfdp_add_usr_msg(self._param_addr, msg)

	@utils._chk_is_open
	def add_filestore_request(self, action, file1, file2=None):
		""" Add a user message to all CFDP PDUs in the next transaction

			:param action: See ``pyion.CFDP_CREATE_FILE``, etc.
			:param file1: String or Path-object
			:param file2: None, string or Path-object.
		"""
		# Transform files to string
		file1 = str(file1)
		if file2 is not None: file2 = str(file2)

		# Call pyion function
		_cfdp.cfdp_add_filestore_request(self._param_addr, action, file1, file2)

	def register_event_handler(self, event, func):
		""" Register and event handler for this entity

			:param event: See ``pyion.constants.CFDP_CREATE_FILE``, etc.
			:param func: Function handle with the following signature
						 ``def ev_handler(params)``, where params is a
						 dictionary with contents that depend on the type
						 of event (see CCSDS CFDP spec, section 3.5.6)
		"""
		self.event_handers[event] = func

	def wait_for_transaction_end(self, timeout=None):
		""" Blocks the calling thread until the transaction has
			finished or the timeout expires.

			:param timeout: Time to wait in [seconds]
			:return: True if transaction finished successfully
		"""
		# Wait for the end of the transaction
		self._end_transaction.wait(timeout=timeout)

		# Return state of transaction
		return self._ok_transaction

	def _mark_transaction_end(self, success):
		""" Mark that the current CFDP transaction has ended

			:param success: True/False
		"""
		# Mark the transaction as successful
		self._ok_transaction = success

		# Awake all threads that were waiting
		self._end_transaction.set()

		# Reset the event
		self._end_transaction.clear()

	def _monitor_events(self):
		""" Monitor all CFDP events """
		while self.is_open:
			# Get the next event
			evt, ev_params = _cfdp.cfdp_next_event()

			# Create event type class from integer code
			evt = CfdpEventEnum(evt)

			# If no event, continue
			if evt == CfdpEventEnum.CFDP_NO_EVENT:
				continue

			# If you have an event handler for this event type, use it
			try:
				func = self.event_handers[evt]
				func(evt, ev_params)
			except KeyError:
				pass

			# If you have a handler for all events, use it
			try:
				func = self.event_handers[CfdpEventEnum.CFDP_ALL_IND]
				func(evt, ev_params)
			except KeyError:
				pass

			# If transaction finished ok, report it
			if evt == CfdpEventEnum.CFDP_TRANSACTION_FINISHED_IND:
				self._mark_transaction_end(True)

			# If transaction failed, report it
			if evt == CfdpEventEnum.CFDP_ABANDONED_IND:
				self._mark_transaction_end(False)

	def __enter__(self):
		""" Allows an endpoint to be used as context manager """
		return self

	def __exit__(self, exc_type, exc_val, exc_tb):
		""" Allows an endpoint to be used as context manager """
		pass

	def __str__(self):
		return '<Entity: {} ({})>'.format(self.entity_nbr, 'Open' if self.is_open else 'Closed')

	def __repr__(self):
		return '<Entity: {} ({})>'.format(self.entity_nbr, self._param_addr)


#!/usr/bin/python3
#
# Copyright 2015 MarkLogic Corporation
#
# This script takes a JSON object that enumerates a set of artifacts and
# removes those artifacts from the server.
#
# See put-config.py
#
# For example:
#
# python3 delete-config --json /tmp/config.json
#
# TODO
#

__author__ = 'pmb'

import argparse
import logging
import json
import os
import base64
import logging
from requests.auth import HTTPDigestAuth
from marklogic.connection import Connection
from marklogic.models.database import Database
from marklogic.models.forest import Forest
from marklogic.models.permission import Permission
from marklogic.models.privilege import Privilege
from marklogic.models.role import Role
from marklogic.models.server import Server
from marklogic.models.user import User

#logging.basicConfig(level=logging.INFO)

parser = argparse.ArgumentParser()
parser.add_argument("--host", action='store', default="localhost",
                    help="Management API host")
parser.add_argument("--username", action='store', default="admin",
                    help="User name")
parser.add_argument("--password", action='store', default="admin",
                    help="Password")
parser.add_argument("--json", action="store",
                    help="Name of the file containing JSON config")
parser.add_argument('--debug', action='store_true',
                    help='Enable debug logging')
args = parser.parse_args()

if args.debug:
    logging.basicConfig(level=logging.WARNING)
    logging.getLogger("requests").setLevel(logging.WARNING)
    logging.getLogger("marklogic").setLevel(logging.DEBUG)

with open(args.json) as data_file:
    data = json.load(data_file)

conn = Connection(args.host, HTTPDigestAuth(args.username, args.password))

# Delete users
for config in data['users']:
    name = config['user-name']
    user = User.lookup(conn, name)
    if user is not None:
        verb = "Deleting"
        user = User.unmarshal(config)
        print("{0} user: {1}".format(verb, name))
        user.delete(conn)

# Delete privileges
for config in data['privileges']:
    name = config['privilege-name']
    kind = config['kind']
    priv = Privilege.lookup(conn, name, kind)
    if priv is not None:
        verb = "Deleting"
        priv = Privilege.unmarshal(config)
        print("{0} {1} privilege: {2}".format(verb, kind, name))
        priv.delete(conn)

# Delete roles
for config in data['roles']:
    name = config['role-name']
    role = Role.lookup(conn, name)
    if role is not None:
        print("Deleting role: {0}".format(name))
        role = Role(name)
        role.delete(conn)

# Delete servers
for config in data['servers']:
    name = config['server-name']
    group = config['group-name']
    server = Server.lookup(conn, name, group)
    if server is not None:
        verb = "Deleting"
        server = Server.unmarshal(config)
        kind = server.server_type()
        print("{0} {1} server: {2}".format(verb, kind, name))
        server.delete(conn)

# Delete databases
for config in data['databases']:
    name = config['database-name']
    db = Database.lookup(conn, name)
    if db is not None:
        print("Deleting database: {0}".format(name))
        db.delete(connection=conn)

# Delete forests
for config in data['forests']:
    name = config['forest-name']
    f = Forest.lookup(conn, name)
    if f is not None:
        print("Deleting forest: {0}".format(name))
        f.delete(connection=conn)