#!/usr/bin/python3
# -*- coding: utf-8 -*-
#
import argparse, subprocess, Bio, os, sys, shutil, re, time, datetime, socket, random, requests, xmltodict, json, csv, sqlite3, types
from Bio import SeqIO
from Bio import Phylo
from Bio import Entrez
from Bio import AlignIO
from Bio.Blast import NCBIWWW
from Bio.Blast import NCBIXML
from Bio.Blast.Applications import NcbirpsblastCommandline
from os.path import basename, dirname, splitext, split
# To print some terminal output in color
import xml.etree.ElementTree as ET
from phylolib import load_blacklist, load_dictionary, insert_line_breaks, write_dict_to_file, read_file_to_dict, execution_time_str, make_synonym_dictionary, create_sqlite_file, expand_complex_taxa, download_proteins, execute_subprocess, get_phylum_from_NCBI, get_taxon_id_from_NCBI
# To draw the phylogenetic tree
from ete3 import Tree, TreeStyle, TextFace, NodeStyle, SequenceFace, ImgFace, SVGFace, faces, add_face_to_node, NCBITaxa

# Purpose: To programmatically retrieve the species numbers in the non-redundant NCBI protein Database
# using e utilities: https://www.ncbi.nlm.nih.gov/books/NBK25500/#chapter1.Searching_a_Database
#
# Database:
# https://www.ncbi.nlm.nih.gov/taxonomy/
# Sample query via the web form: porifera [subtree] AND specified [prop] NOT subspecies [rank]

# This script goes through all primary blast result files and does the following:
#
# It looks at the protein description and determines which ortholog group it belongs to.
# If this cannot be figured out from the curated protein description using the list of
# aliases from master_dictionary.py (e.g. "HYPOTHETICAL PROTEIN"), then it initiates a
# blastp search ("backcheck blast") with that protein and categorizes the results
# according to the majority description of the 50 best blastp results (THRESHOLD = 0.5)
# into one of three groups: "synonym", "related protein" or "unknown". The unknown
# proteins are written to the file "check_manually.html".
#
# - Determines the number of species for each taxon in the NCBI database
# - Determines the number of fully sequenced genomes of the taxons specified in the taxon_dictionary.py file
# - Updates the taxon_dictionary.py file with the results

# The results of all tasks that require to connect to a remote service or database (NCBI, Entrez) already
# cached in order for subsequent runs to be faster. Only after deleting the local cache (manually by deleting
# the files), new remote requests are initiated.

# PREREQUISITS
# Ubuntu 16.04 or 18.04
#
# sudo apt -y --show-progress installpython3-pip t-coffee inkscape python3-setuptools python3-pyqt4 python3-pyqt4.qtopengl python3-pip autoconf imagemagick build-essential libblas-dev liblapack-dev zlib1g-dev libcairo2-dev libcurl4-openssl-dev python3-numpy python3-lxml python3-six
# sudo pip3 install biopython xmltodict ete3
#
# These links need to be set:
# sudo ln -s /usr/bin/dialign-tx /bin/dialign-t
# sudo ln -s /usr/bin/clustalw /bin/clustalw2
#
# The pcma executable needs to be installed manually as follows:
# wget http://prodata.swmed.edu/download/pub/PCMA/pcma.tar.gz
# tar -xvzf pcma.tar.gz
# cd pcma
# make
# sudo cp pcma /bin
#
# If you do not install the pygt4 packages, you get a really
# uninformative error message:
# "ImportError: cannot import name 'TreeStyle':"

class category_found(Exception):
    pass

class species_number_error(Exception):
    pass

# Parse command line
parser = argparse.ArgumentParser()
parser.add_argument("--directory", nargs='?', help = "Specify the subdirectory where the data is!", default = 'data/primary_blast_results/')
args = parser.parse_args()

# This will conenct to NCBI taxonomy database and retrieve the number of species for a given phylum
# If the NCBI database is down, this will loop forever!
def get_species_number_from_ncbi(taxon, taxon_data):
    print('Retrieving species number for {0}... -> '.format(taxon), end='')
    # Determine, whether a taxon is to be excluded from the request
    TAXA_LIST = expand_complex_taxa(taxon_data[0])
    while True:
        try:
            number = 0
            for item in TAXA_LIST:
                sign = item[0]
                URL = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=taxonomy&term=txid{0}[subtree]+AND+specified[prop]+NOT+subspecies[rank]'.format(item[1:])
                print('URL: {0}'.format(URL))
                r = requests.get(URL)
                text = r.text
                print('species_number_text:\n{0}'.format(text))
                if "<ERROR>" in text:
                    raise species_number_error
                # Add only the first number and substract all following numbers
                if sign == '+':
                    number += int(text.split("Count>")[1][:-2])
                if sign == '-':
                    number -= int(text.split("Count>")[1][:-2])
        except species_number_error:
            print('Remote database error. Waiting 30 seconds and repeating request.')
            time.sleep(30)
        except Exception as err:
            print('Failed to fetch species number. Error: {0}'.format(err))
        else:
            # the "else" is only executed if everything under the "try" has completed without error
            break
    return number

def get_taxon_id_and_phylum(species_name):
    # Check whether the species is in the local sqlite database
    print('Getting id & phylum for \'{0}\', checking local sqlite database first.\n'.format(species_name), end = '')
    if species_name in sqlite_species_dict:
        TAXON_ID = sqlite_species_dict[species_name][1]
        PHYLUM = sqlite_species_dict[species_name][2]
    else:
        print("Species not in local sqlite database. Getting data from NCBI... ")
        TAXON_ID = get_taxon_id_from_NCBI(species_name)
        print('=> {0}'.format(TAXON_ID))
        # Identify to which phylum the species belongs. For this, we need to download the full taxonomy tree for the species (which
        # is impossible via the API). Hence, we retrieve the full web page for the taxon and look whether any of the phyla from our
        # TAXON_DICTIONARY_FILE are part of the full taxonomy tree.
        PHYLUM = get_phylum_from_NCBI(TAXON_ID)
        print('=> {0}'.format(PHYLUM))
        if db_insert_species(species_name, TAXON_ID, PHYLUM) != 0:
            print('New species {0} inserted into database.'.format(species_name))
    return TAXON_ID, PHYLUM

# How many sequences are in the local database for this taxon? The gi files were manually downloaded from
# NCBI. E.g. https://www.ncbi.nlm.nih.gov/protein/?term=txid7777%5Borganism%5D and then "Send to file" -> "Format: GI List" -> "Create File"
# There should be smarter way to get these!
def get_sequence_number(taxon, taxon_data):
    print('Retrieving number of sequences for taxon {0}... -> '.format(taxon), end='')
    TAXA_LIST = expand_complex_taxa(taxon_data[0])
    i = 0
    for item in TAXA_LIST:
        SEQUENCE_GI_LIST = '{0}/data/gi_lists/txid{1}.gi'.format(APPLICATION_PATH, item[1:])
        with open(SEQUENCE_GI_LIST) as file:
            for j, m in enumerate(file):
                pass
        if item[0] == '+':
            i += j
        elif item[0] == '-':
            i -= j
    return i + 1

def make_tree(ALIGNMENT_FILE, MODE):
    treedir = '{0}/data/alignments_and_trees'.format(APPLICATION_PATH)
    os.chdir(treedir)
    # Detect whether parallel bootstrapping should be performed
    mpirun_path = shutil.which('mpirun')
    phymlmpi_path = shutil.which('phyml-mpi')
    if MODE == 'mcoffee':
        # -5 = approximate Bayes branch supports
        BRANCH_SUPPORT = '-5'
        SEARCH_OPERATION = 'BEST'
    elif MODE == 'fmcoffee':
        BRANCH_SUPPORT = '-1'
        SEARCH_OPERATION = 'SPR'
    else:
        # for large datasets, do not calculate the branch support (MODE = 'quickaln')
        BRANCH_SUPPORT = 0
        SEARCH_OPERATION = 'NNI'
    if mpirun_path != '' and phymlmpi_path != '':
        # Run on multiple cores
        phylo_command = 'mpirun -n 4 phyml-mpi --no_memory_check -s {0} -i {1} -d aa -b {2}'.format(SEARCH_OPERATION, ALIGNMENT_FILE, BRANCH_SUPPORT)
    else:
        # Run on single core (do not )
        phylo_command = 'phyml --no_memory_check -s {0} -i {1} -d aa -b -1'.format(SEARCH_OPERATION, ALIGNMENT_FILE)

    # The gene tree building is actually never used since the species tree is used for the tree drawing.
    # We anyway calculate it to be able to compare gene and species trees.
    result, error = execute_subprocess('Make phylogenetic tree with the following command: ', phylo_command)
    return result, error

def write_to_html_scrutinize_file(protein, taxon, list_to_scrutinize):
    # This sorts the list of lists according to the third element of each list (= type of hit; synonym, related protein, unknown)
    list_to_scrutinize.sort(key=lambda x: x[2])
    preamble = '''

<title>Results</title>

'''
    #for item in sorted_list:
    new_type = ''
    i = 1
    #j = 1 not needed
    html_text = ''
    print('list_to_scrutinize:\n{0}'.format(list_to_scrutinize))
    for item in list_to_scrutinize:
        # Make a heading row if the type of hit changes and color unknown stuff red
        if item[2] != new_type:
            if item[2] in ['hypothetical protein', 'uncharacterized protein', 'unknown']:
                red_color = ' style="color:Red;"'
            else:
                red_color = ''
            if item[2] != '':
                # End of toggle visibility section
                html_text += '
\n
\n\n'
            html_text += '
{4}. {1}-{2}-{3}
\n'.format(red_color, item[0], item[1], item[2], i)
            # Start of toggle visibility section
            html_text += '
\n'.format(item[0], item[1], item[2])
            # New table
            #lineitem += '
\n

{0}

{1}

{2}

\n'.format(item[0], item[1], item[2])
            html_text += '
\n'.format()
        link_to_protein = '{1}'.format(item[3], item[5])
        link_to_blast = 'blastp'.format(item[3])
        link_to_local_msa = 'local MSA'.format(taxon, item[3])
        #                                                                                taxon    protein  type_of  id       accesion_no
        #                                                                                _hit
        html_text += '
{0}	{1}	{2}	{3}	{4}	{5}	{6}	{7}
\n'.format(item[0], item[1], item[2], item[3], item[4], link_to_protein, link_to_blast, link_to_local_msa)
        new_type = item[2]
        i += 1
    #lineitem += '\n\n\n'
    HTML_DIR = '{0}/data/analysis_results/{1}'.format(APPLICATION_PATH, protein)
    HTML_FILE = '{0}/{1}.html'.format(HTML_DIR, taxon)
    if not os.path.isdir(HTML_DIR):
        os.mkdir(HTML_DIR)
    if not os.path.isfile(HTML_FILE):
        with open(HTML_FILE, 'w') as file:
            file.write(preamble)
            print('\nCreated file {}.'.format(HTML_FILE))
    with open(HTML_FILE, 'a') as file:
        file.write(html_text)
        print('\nWrote HTML file {0}.'.format(HTML_FILE))

def add_to_protein_hitdict(taxon, protein, hit_id, hit_description, ortholog_group, hit_accession_no):
    global protein_hitdict
    # Do not write the same protein twice, but add the protein to the string
    print('Adding to protein_hitdict...')
    if hit_accession_no in protein_hitdict:
        protein_hitdict[hit_id][3] += ' '+ortholog_group
        print('{0} added to existing protein.'.format(hit_id))
    else:
        protein_hitdict[hit_id] = [taxon, protein, hit_description, ortholog_group, hit_accession_no]
        print('{0} added as new protein.'.format(hit_id))

def most_frequent(List):
    return max(set(List), key = List.count)

def write_protein_hitdict_to_file(protein_hitdict):
    global TRUE_UNIQUES, SVG_DICT
    print('Writing protein files and doing multiple sequence alignments based on protein_hitdict with {0} unique sequences.'.format(len(protein_hitdict)))
    for key, value in protein_hitdict.items():
        # Get the taxon name from somewhere (all entries in the dictionary should have the same taxon in value[0])
        taxon = value[0]
        print('Ecluding proteins. value = {0}'.format(value))
        # Do not deal with excluded proteins at all!
        if os.path.isfile('{0}/data/proteins_exclude/{1}.fasta'.format(APPLICATION_PATH, key)):
            print('Protein {0} was manually excluded. Skipping...'.format(key))
        # Check also in the taxon-specific subfolder
        elif os.path.isfile('{0}/data/proteins_exclude/{1}/{2}.fasta'.format(APPLICATION_PATH, taxon, key)):
            print('Protein {0} was found in subfolder {1} and manually excluded. Skipping...'.format(key, value[0]))
        else:
            print('Including protein {0} as it was not found in {1}/data/proteins_exclude or {1}/data/proteins_exclude/{2}'.format(key, APPLICATION_PATH, value[0]))
            print('{0} -> {1}'.format(key, value))
            # Evaluate which protein was mostly identified as a homolog (in order to include it in the MSA)
            closest_homolog_list = value[3].split()
            # Delete all "None" elements from the list
            closest_homolog_list = list(filter(lambda x: x!= 'None', closest_homolog_list))
            if len(closest_homolog_list) > 0:
                closest_homolog = most_frequent(closest_homolog_list)
                print('Closest homolog: {0}'.format(closest_homolog))
                if closest_homolog != 'manually_excluded':
                    QUERY_FILE_CLOSEST_HOMOLOG = '{0}/data/reference_proteins/{1}.fasta'.format(APPLICATION_PATH, master_dictionary[closest_homolog][0])
                else:
                    print('{0} was manually excluded from analysis'.format(key))
                    # Default comparison to VEGF-C
                    QUERY_FILE_CLOSEST_HOMOLOG = '{0}/data/reference_proteins/NP_005420.1.fasta'.format(APPLICATION_PATH)
            else:
                #closest_homolog = ''
                print('No closest homolog identified.')
                # Default comparison to VEGF-C
                QUERY_FILE_CLOSEST_HOMOLOG = '{0}/data/reference_proteins/NP_005420.1.fasta'.format(APPLICATION_PATH)
            PROT_FILE = '{0}/data/protein_results/{1}.html'.format(APPLICATION_PATH, value[0])
            QUERY_FILE = '{0}/data/proteins/{1}.fasta'.format(APPLICATION_PATH, key)
            MATRIX_FILE = '{0}/data/GONNETmod.matrix'.format(APPLICATION_PATH)
            if not os.path.isfile(QUERY_FILE) or os.stat(QUERY_FILE).st_size == 0:
                download_proteins('{0}/data/proteins'.format(APPLICATION_PATH), {key: [key]})
                time.sleep(1)
            VEGFC_signature = '{0}/data/reference_proteins/VEGFC_signature.fasta'.format(APPLICATION_PATH)
            # Simple one-to-one comparison
            #bash_command = 'needle {0} {1} -gapopen 10 -gapextend 0.5 stdout'.format(QUERY_FILE1, QUERY_FILE2)
            # MSA including additonally the VEGF signature uding edialign
            #bash_command = 'cat {0} {1} {2} | edialign -filter'.format(QUERY_FILE1, QUERY_FILE2, VEGFC_signature)
            # MSA using emma (clustalx)
            #
            # COMMAND TO USE FOR CUSTOM MATRIX THAT OVEREMPHASIZES CYSTEIN ALIGNMENTS:
            bash_command = 'cat {0} {1} {2}| emma -filter -slowalign Yes -pwmatrix o -pairwisedatafile {3} -osformat2 msf -dendoutfile /dev/zero'.format(QUERY_FILE, VEGFC_signature, QUERY_FILE_CLOSEST_HOMOLOG, MATRIX_FILE)
            #bash_command = 'cat {0} {1} {2}| emma -filter -slowalign Yes -pwmatrix g -osformat2 msf -dendoutfile /dev/zero'.format(QUERY_FILE, VEGFC_signature, QUERY_FILE_CLOSEST_HOMOLOG)
            comment = 'Making alignment of {0}, {1} and VEGF-C signature:\n'.format(key, value[1])
            alignment, error = execute_subprocess(comment, bash_command)
            # Trim header from msf file (necessary for emma)
            alignment = alignment.split('//')[1]
            # Trim comments from the alignment text blob (necessary for needle)
            line_list = alignment.split('\n')
            alignment = ''
            for line in line_list:
                if not line.startswith('#'):
                    alignment += line+'\n'
            # Write html header if the file is newly created
            if not os.path.exists(PROT_FILE):
                with open(PROT_FILE, 'w') as handle:
                    print('Writing header of html file {}'.format(PROT_FILE))
                    preamble = '''

                    <title>Individual alignments and phylogenetic tree for taxon '''
                    preamble += taxon
                    preamble += '''</title>

'''
                    preamble += taxon
                    preamble += '''

'''
                    handle.write(preamble)
            # This writes the actual rows in the table
            with open(PROT_FILE, 'a') as handle:
                # The value does not contain the Acc no....
                link_to_protein = '{0}'.format(key)
                link_to_blast = 'blastp'.format(key)
                row = '
{1}	{2}	{3}	{4}	{5}	{6}	alignment'.format(key, link_to_protein, value[4], value[1], value[2], value[3], link_to_blast)
                row += '

\n{1}\n

\n'.format(key, alignment)
                SVG_DICT[taxon][value[4]] = value[1], value[2], value[3]
                print('SVG_DICT creationtime: {0}'.format(SVG_DICT))
                handle.write(row)

    # Make MSA for each taxon (limit by sequence number)
    count_taxa = 0
    number_of_taxa = len(taxon_dictionary)
    for taxon in taxon_dictionary:
        count_taxa += 1
        # The protein_hitdict comprises all (~8666) sequences!!!!
        # Use list comprehension to extract all sequences of a certain taxon
        #taxon_specific_protein_hitlist = [key for key, value in protein_hitdict.items() if value[1] == taxon]
        taxon_specific_protein_hitlist = []
        print('\nAppending to taxon_specific_protein_hitlist ({0}):'.format(taxon))
        for key, value in protein_hitdict.items():
            #print('taxon: {0}'.format(taxon))
            #print('value: {0}'.format(value))
            if value[0] == taxon:
                taxon_specific_protein_hitlist.append(key)
                print('{0}-{1} '.format(value[0], key), end = '')
        how_many_specific = len(taxon_specific_protein_hitlist)
        print('\n\ntaxon_specific_protein_hitlist ({0} proteins):'.format(how_many_specific))
        # If LIMIT['MAX'] or more sequences are present, the MSA is skipped
        # with LIMIT['MAX'] = 103, 102 (= crocodylia) works ok, but arachnida (= 103) is skipped)
        LIMIT = {'ACCURATE': 25, 'SEMIACCURATE': 104, 'MAX': 200}
        #LIMIT = {'ACCURATE': 5, 'SEMIACCURATE': 10, 'MAX': 50}
        # Only printing
        if how_many_specific < LIMIT['MAX']:
            for i in range(1, how_many_specific+1):
                print('{0}.\n{1}'.format(i, taxon_specific_protein_hitlist[i-1]))
        else:
            # Might need fixing for the exact numbers
            print('1.\n{0}\n2.-{1}. [...]'.format(taxon_specific_protein_hitlist[0], how_many_specific-LIMIT['MAX']))
            for i in range(how_many_specific-LIMIT['MAX']+1, how_many_specific+1):
                print('{0}.\n{1}'.format(i, taxon_specific_protein_hitlist[i-1]))
        #
        # HERE THE BLOCK STARTS FOR DOING THE ALIGNMENTS AND TREESEARCH
        #
        # Set lower computational requirements for large datasets (for both alignment and tree search)
        if how_many_specific < LIMIT['ACCURATE']:
            # Do alignment using the slow mcoffee option
            MODE = 'mcoffee'
        elif how_many_specific < LIMIT['SEMIACCURATE']:
            # Do alignment using the faster fmcoffee option
            MODE = 'fmcoffee'
        else:
            # Do alignment using the even faster quickaln option
            MODE = 'quickaln'
        if 0 < how_many_specific < LIMIT['MAX']:
            number_of_reference_sequences = len(master_dictionary)
            generate_tree = True
            # Fix filenames with whitespaces (because t_coffee cannot handle them)
            taxon_sanitized = "_".join(taxon.split())
            # Name of the html output file from t_coffee
            outputfile_html = '{0}/data/alignments_and_trees/{1}_all.html'.format(APPLICATION_PATH, taxon_sanitized)
            # Name of the clustalw output file from t_coffee
            MSA_outputfile_from_tcoffee = '{0}/data/alignments_and_trees/{1}_all.clustalw'.format(APPLICATION_PATH, taxon_sanitized)
            # Name of the converted phylip output file from biopython
            outputfile_phylip = '{0}/data/alignments_and_trees/{1}_all.phylip'.format(APPLICATION_PATH, taxon_sanitized)
            # Do only (during testing) if the alignment file does not yet exist!
            if not os.path.isfile(MSA_outputfile_from_tcoffee):
                #print('taxon_specific_protein_hitlist:\{0}'.format(taxon_specific_protein_hitlist))
                alignment_file_list = []
                # Add only sequences to the MSA list that exist and that are not in the excluded list!
                TRUE_UNIQUES[taxon] = 0
                for id in taxon_specific_protein_hitlist:
                    seqfile = '{0}.fasta'.format(id)
                    if os.path.isfile('{0}/data/proteins_exclude/{1}'.format(APPLICATION_PATH, seqfile)) or os.path.isfile('{0}/data/proteins_exclude/{1}/{2}'.format(APPLICATION_PATH, taxon, seqfile)):
                        print('Sequence {0} was manually excluded. Omitting from MSA for {1}'.format(seqfile, taxon))
                    elif os.path.isfile('{0}/data/proteins/{1}'.format(APPLICATION_PATH, seqfile)):
                        alignment_file_list.append(seqfile)
                        TRUE_UNIQUES[taxon] += 1
                        print('TRUE_UNIQUES: {0}'.format(TRUE_UNIQUES))
                    else:
                        print('Sequence {0} not found. Omitting from MSA for {1}'.format(seqfile, taxon))
                print('Final TRUE_UNIQUES: {0}'.format(TRUE_UNIQUES))
                print('Number of homologs included in the MSA: {0}'.format(TRUE_UNIQUES[taxon]))
                # This adds all the reference proteins to the MSA list
                # Also count reference sequences!
                number_of_reference_sequences = 0
                for key, value in master_dictionary.items():
                    seqfile = '../reference_proteins/{0}.fasta'.format(value[0])
                    if os.path.isfile('{0}/data/reference_proteins/{1}.fasta'.format(APPLICATION_PATH, value[0])):
                        alignment_file_list.append(seqfile)
                        number_of_reference_sequences += 1
                # Only execute MSA if there are non-reference sequences in the alignment_file_list
                if len(alignment_file_list) > number_of_reference_sequences:
                    print('FOR taxon {0}: len(alignment_file_list) = {1} > number_of_reference_sequences = {2}'.format(taxon, len(alignment_file_list), number_of_reference_sequences))
                    # Using emboss/emma (clustalx)
                    #bash_command = 'cat {0} | emma -filter -osformat2 msf -dendoutfile /dev/zero'.format(alignment_file_list)
                    # Using m_coffee and html output
                    #
                    # Concatenate all fasta files (t_coffee crashes when using too many direct input files)
                    concat_fasta_file = '../protein_results/{0}_all.fasta'.format(taxon_sanitized)
                    # Convert list of files into space-separated string
                    alignment_file_list_str = ' '.join(alignment_file_list)
                    bash_command = 'cat {0} > {1}'.format(alignment_file_list_str, concat_fasta_file)
                    comment = 'Concatenating all fasta files for taxon {0}'.format(taxon)
                    result, error = execute_subprocess(comment, bash_command, working_directory='{0}/data/proteins/'.format(APPLICATION_PATH))
                    # The nice -n 19 is because I have to work on the same computer!
                    # Need to output into something different than phylip since t_coffee uses the strict phylip format
                    # which does not allow for more than 10 characters in the sequence name; try clustalw
                    bash_command = 'nice -n 19 t_coffee -seq {0} -output=html,clustalw -mode {1}'.format(concat_fasta_file, MODE)
                    comment = 'Making MSA for {0} VEGFs/PDGFs (alignment #{1} from total {2})\n'.format(taxon, count_taxa, number_of_taxa)
                    result, error = execute_subprocess(comment, bash_command, working_directory='{0}/data/alignments_and_trees'.format(APPLICATION_PATH))
                    #print('Result of MSA subprocess: {0}'.format(result))
                    # Convert clustalw format into phylip format for Phyml
                    input_alignment = open(MSA_outputfile_from_tcoffee, "rU")
                    output_alignment = open(outputfile_phylip, "w")
                    alignments = AlignIO.parse(input_alignment, "clustal")
                    AlignIO.write(alignments, output_alignment, "phylip-relaxed")
                    output_alignment.close()
                    input_alignment.close()
                else:
                    print('Not generating MSA since number of homologous sequences in taxon {0} is 0.'.format(taxon))
                    generate_tree = False
                    with open(outputfile_html, 'w') as file:
                        file.write('MSA was not generated since number of homologous sequences in taxon {0} is 0.'.format(taxon))
            else:
                # Take the number of true uniques from the phylip alignment file (= 1st number in the first row of the phylip file)
                with open(outputfile_phylip, 'r') as file:
                    TRUE_UNIQUES[taxon] = int(file.readline().split()[0]) - number_of_reference_sequences

            # Do only (during testing) if the tree file does not yet exist!
            treefile = '{0}_phyml_tree.txt'.format(outputfile_phylip)
            if not os.path.isfile('{0}/data/alignments_and_trees/{1}_all.phylip_phyml_tree.txt'.format(APPLICATION_PATH, taxon_sanitized)):
                if generate_tree == True:
                    result, error = make_tree(outputfile_phylip, MODE)
                else:
                    with open(treefile, 'w') as file:
                        file.write('Phylogenetic tree was not generated since number of homologous sequences in taxon {0} is 0.'.format(taxon))
                        generate_tree == False
            # Read tree file and append to 'html_string' string
            #with open(treefile, 'r') as file:
            #    html_string += '

{0}'.format(file.read())

            # Check that the Newick tree file exists and that it contains a tree!
            if os.path.isfile('{0}/data/alignments_and_trees/{1}_all.phylip_phyml_tree.txt'.format(APPLICATION_PATH, taxon_sanitized)) and generate_tree == True:
                #Draw tree
                print('Drawing phylogenetic tree for taxon {0}'.format(taxon))
                TREE_IMAGE_SVG = '{0}/data/alignments_and_trees/{1}.svg'.format(APPLICATION_PATH, taxon_sanitized)
                # Convert phylip into newick format

                t = Tree(treefile)
                ts = TreeStyle()
                ts.show_leaf_name = False
                # Zoom in x-axis direction
                ts.scale = 20
                # This makes all branches the same length!!!!!!!
                ts.force_topology = True
                #t.set_outgroup(t & OUTGROUP[0])
                ts.show_branch_support = False
                ts.show_branch_length = False
                ts.draw_guiding_lines = True
                ts.branch_vertical_margin = 10 # 10 pixels between adjacent branches
                # Put protein description to tree leaves
                # Phylip limits sequence names to 10 characters, which creates again problems....
                print('SVG_DICT[taxon] usetime: {0}'.format(SVG_DICT[taxon]))
                for node in t.traverse():
                    if node.is_leaf():
                        # Because of the acc_no / id mixup, the SVG_DICT is necessary!
                        print('node.name (taxon): {0} ({1})'.format(node.name, taxon))
                        sanitized_nodename = node.name.split('.')[0]
                        if sanitized_nodename in SVG_DICT[taxon]:
                            print('node.name: {0}; values: {1}'.format(node.name, SVG_DICT[taxon][sanitized_nodename]))
                            textFace = TextFace(SVG_DICT[taxon][sanitized_nodename], fsize = 10)
                            (t & node.name).add_face(textFace, 0, "aligned")
                Tree.render(t, TREE_IMAGE_SVG)
                # Embed tree image to html file
                with open(treefile, 'a') as file:
                    html_string = '
<object type="image/svg+xml" data="../alignments_and_trees/{0}.svg">Here should be the svg tree!</object>
'.format(taxon_sanitized)
            else:
                print('Not drawing phylogenetic tree for taxon {0}'.format(taxon))
                html_string += '
No tree image!
'
            # Add alignment file to html file
            with open(outputfile_html, 'r') as file:
                html_string += file.read()
        else:
            if how_many_specific == 0:
                TRUE_UNIQUES[taxon] = '0'
                print('Not generating MSA/tree since number of homologous sequences in taxon {0} is 0.'.format(taxon))
                html_string = 'MSA/tree were not generated since number of homologous sequences in taxon {0} is 0.'.format(taxon)
            else:
                TRUE_UNIQUES[taxon] = 'n.a.'
                print('Not generating MSA/tree since number of homologous sequences in taxon {0} is too high ({1}).'.format(taxon, how_many_specific))
                html_string = 'MSA/tree were not generated since number of homologous sequences in taxon {0} ({1}) exceeded the limit of {2}.'.format(taxon, how_many_specific, LIMIT['MAX'])
        PROT_FILE = '{0}/data/protein_results/{1}.html'.format(APPLICATION_PATH, taxon)
        with open(PROT_FILE, 'a') as handle:
            # Don't make alignments for large numbers of proteins
            end_of_file = '
\n'
            end_of_file += '
\n{0}\n
\n'.format(html_string)
            end_of_file += '\n'
            handle.write(end_of_file)
            print('Writing MSA for taxon {0} to {1}'.format(taxon, PROT_FILE))

def download(sequence_list, save_in_file_or_folder):
    # Get all fasta sequences from the SEQUENCE_LIST via Entrez
    Entrez.email = "michael@jeltsch.org"
    Entrez.tool = "local_script_under_development"
    file = open(save_in_file_or_folder,"w")
    print("Retrieving sequences from Entrez:\n")
    for item in sequence_list:
        print("Retrieving " + item)
        try:
            with Entrez.efetch(db="protein", rettype="fasta", retmode="text", id=item) as handle:
                seq_record = SeqIO.read(handle, "fasta")
                file.write(seq_record.format("fasta"))
        except Exception as ex:
            print("Problem contacting Blast server. Skipping " + item + ". Error: " + str(ex))
    file.close()

def get_protein_data(taxon):
    global TOTAL_COUNT, TOTAL_UNKNOWN_COUNT, conn, excluded_list, TRUE_UNIQUES
    if args.directory == '':
        return []
    else:
        new_protein_data = {}
        print('\n---------------')
        print('Analyzing taxon {0}'.format(taxon).upper())
        print('---------------\n')
        for protein, protein_data in master_dictionary.items():
            list_to_scrutinize_ortholog = []
            list_to_scrutinize_paralog = []
            list_to_scrutinize_unknown = []
            list_to_scrutinize_manually_excluded = []
            #print('\n\nSTART\nPROTEIN: {0}\nTAXON: {1}\n\n'.format(protein, taxon))
            #print('master_dictionary:\n{0}\n\n'.format(master_dictionary))
            #time.sleep(2)
            # Here we analyze the blasts hits for each TAXON/PROTEIN pair:
            # a) number of blast hits
            # b) number of false positive according to the description (categorized in a list)
            #
            # Open the individual results file for a protein and a phylum
            XML_RESULTS_FILE = '{0}{1}/blast_results_{2}.xml'.format(args.directory, protein, taxon)
            # Create database connection
            #conn = create_connection(DATABASE_FILE)

            # Get extended information about the false-positive hits
            negative_dict = {}
            for related_protein in protein_data[4]:
                negative_dict[related_protein] = 0
            # Get extended information about the true-positive hits
            positive_dict = {}
            for synonym in protein_data[3]:
                positive_dict[synonym] = 0

            print('\nAnalyzing {0} ({1}):'.format(protein, taxon).upper())
            blastp_result = SearchIO.read(XML_RESULTS_FILE, "blast-xml")
            number = len(blastp_result)
            print('Number of blastp results: {0}'.format(number))
            # counter for hits
            i = 0
            # counter for unknown proteins
            unknown_counter = 0
            # counter for manually excluded proteins
            excluded_counter = 0
            for hit in blastp_result:
                found = False
                # get id and accession number
                # Blast has changed the data it returns in the Hit_id field of the xml file.
                # It used to report both id and accession number, but now returns only the
                # the accession number appended with the version (e.g. "ref|XP_015200204.1|")
                # the Hit_accession field of the xml file contains the accession number without
                # the version extension
                hit_id = hit.id.split('|')[1]
                hit_accession_no = hit.accession
                print('Analyzing #{0} (from {1} {2} {3} homologs): hit_id:{4} - hit_accession_no:{5}'.format(i+1, number, taxon, protein_data[3][0], hit_id, hit_accession_no))
                # Extract species name; some entries are not according to the specs
                # and we need to handle that...
                try:
                    species = re.search(r'\[(.*?)\]',hit.description).group(1)
                except Exception as err:
                    print('Non-standard protein description ({0}). Continuing without species information...'.format(hit.description))
                    species = 'unknown'
                #print('Checking false- or true-positivity.')
                # If one protein category matches, we need to abort searching to avoid items being counted twice or more often if
                # their description contains a repetition like "VEGF-A (Vacular endothelial growth factor-A)".
                # The Try statement is simply for escaping both for-loops
                ortholog_group = 'None'
                try:
                    # Try synonyms first (more likely to find a category?)
                    for synonym in protein_data[3]:
                        if synonym.lower() in str(hit.description).lower():
                            positive_dict[synonym] += 1
                            print('True positive found: {0}'.format(synonym))
                            #list_to_scrutinize.append([taxon, protein, synonym, hit_id, hit_accession_no, hit.description])
                            ortholog_group = protein_data[3][0]
                            list_to_scrutinize_ortholog.append([taxon, protein, 'ortholog', hit_id, hit_accession_no, hit.description])
                            raise category_found
                    for related_protein in protein_data[4]:
                        if related_protein.lower() in str(hit.description).lower():
                            negative_dict[related_protein] += 1
                            print('Potential false-positive found: {0}'.format(related_protein))
                            #list_to_scrutinize.append([taxon, protein, related_protein, hit_id, hit_accession_no, hit.description])
                            # related_protein should always give the same name!!!!
                            # This is why we need the synonym dictionary
                            try:
                                ortholog_group = synonym_dictionary[related_protein]
                                list_to_scrutinize_paralog.append([taxon, protein, 'paralog', hit_id, hit_accession_no, hit.description])
                            except Exception as err:
                                ortholog_group = 'None'
                                list_to_scrutinize_paralog.append([taxon, protein, 'paralog', hit_id, hit_accession_no, hit.description])
                            raise category_found
                    # This part of the Try block gets only executed when both for-loops are finshing without
                    # raising a category_found exception
                    what_kind_of_protein = run_backcheck_blast(hit_accession_no, protein_data)
                    if what_kind_of_protein == 'synonym':
                        positive_dict[synonym] += 1
                        TOTAL_COUNT += 1
                        print('True positive found after backcheck blast: {0}'.format(protein_data[3][0]))
                        ortholog_group = protein_data[3][0]
                        list_to_scrutinize_ortholog.append([taxon, protein, 'ortholog', hit_id, hit_accession_no, hit.description])
                    elif what_kind_of_protein == 'related_protein':
                        negative_dict[related_protein] += 1
                        TOTAL_COUNT += 1
                        #print('Potential false-positive found: {0}'.format(related_protein))
                        print('Potential false-positive found.')
                        ortholog_group = 'None'
                        list_to_scrutinize_paralog.append([taxon, protein, 'paralog', hit_id, hit_accession_no, hit.description])
                    elif '{0}.fasta'.format(hit_id) in excluded_list:
                        print('Manually excluded sequence encountered: {0}/{1}'.format(hit_id, hit_accession_no))
                        list_to_scrutinize_manually_excluded.append([taxon, protein, 'manually_excluded', hit_id, hit_accession_no, hit.description])
                        ortholog_group = 'manually_excluded'
                        # Subtract manually false-positives (that are not homologous to VEGFs/PDGFs) from the total number of hits
                        number -= 1
                        excluded_counter += 1
                        TOTAL_COUNT += 1
                    else:
                        print('rpsblast pathway')
                        # These are all the unknown proteins, that even a backcheck blast cannot identify
                        # Do an RPS BLAST searching for the PDGF signature and classify as related protein if found
                        # Get the profile database from ftp://ftp.ncbi.nih.gov/pub/mmdb/cdd/cdd.tar.gz
                        # Make a file "pdgf.pn", which contains the filenames of the profiles that you want to search for.
                        # In our case (PDGF domain) that is smart00141.smp and cd00135.smp
                        # makeprofiledb -title PDGF_V1.0 -in pdgf.pn -out PDGF -threshold 9.82 -scale 100.0 -dbtype rps -index true
                        # rpsblast -query NP_079484.1.fasta -db PDGF -evalue 0.00001 -outfmt 5 -out out.xml
                        # Download the sequence from Entrez
                        query_filename = '{0}/rpsblast/data/{1}.fasta'.format(APPLICATION_PATH, hit_accession_no)
                        if not os.path.isfile(query_filename):
                            download([hit_accession_no], query_filename)
                        xml_result_filename = '{0}/rpsblast/data/{1}.xml'.format(APPLICATION_PATH, hit_accession_no)
                        # Adjust the file locations:
                        rpsblast_database = "rpsblast/PDGF"
                        rpsblast_executable = '/home/mjeltsch/bin/ncbi-blast-2.9.0+/bin/rpsblast'
                        #Adjust the expectation cut-off here
                        E_VALUE_THRESHOLD = 0.00001
                        # THIS DOES NOT WORK IF THE Ubuntu-provided blast version is used!
                        # rpsblast_cline = NcbirpsblastCommandline(cmd=rpsblast_executable, query=query_filename, db=rpsblast_database, evalue=E_VALUE_THRESHOLD, outfmt=5, out=xml_result_filename)
                        # try:
                        #     stdout, stderr = rpsblast_cline()
                        # except Exception as ex:
                        #     print("\nSomething went wrong with the local rpsblast. More about the error:\n" + str(ex))
                        bash_command = '{0} -query {1} -db {2} -evalue {3} -outfmt 5 -out {4}'.format(rpsblast_executable, query_filename, rpsblast_database, E_VALUE_THRESHOLD, xml_result_filename)
                        comment = 'Executing rpsblast...'
                        result, error = execute_subprocess(comment, bash_command, working_directory='.')
                        print('rpsblast result: {0}'.format(result))

                        # Open XML results for analysis
                        for record in NCBIXML.parse(open(xml_result_filename)):
                            #We want to ignore any queries with no search results:
                            if record.alignments:
                                print('rpsblast found at least one PDGF homology domain:')
                                print('QUERY: {0}...'.format(record.query[:60]))
                                for align in record.alignments :
                                    for hsp in align.hsps :
                                        print('{0} HSP, e={1}, from position {2} to {3}'.format(align.hit_id, hsp.expect, hsp.query_start, hsp.query_end))
                                        assert hsp.expect <= E_VALUE_THRESHOLD
                                #negative_dict[related_protein] +=after_value 1
                                #TOTAL_COUNT += 1
                                #print('Potential false-positive found.')
                                #ortholog_group = 'None'
                                #list_to_scrutinize_paralog.append([taxon, protein, 'paralog', hit_id, hit_accession_no, hit.description])
                                # Above 5 lines were disabled for debugging and replaced by below 5 lines
                                hit.description += ', PDGF signature programmatically detected'
                                list_to_scrutinize_unknown.append([taxon, protein, 'unknown', hit_id, hit_accession_no, hit.description])
                                ortholog_group = 'None'
                                unknown_counter+= 1
                                TOTAL_COUNT += 1
                                TOTAL_UNKNOWN_COUNT += 1
                            else:
                                print('rpsblast found no PDGF homology domains.')
                                hit.description += ', no PDGF signature programmatically detected'
                                list_to_scrutinize_unknown.append([taxon, protein, 'unknown', hit_id, hit_accession_no, hit.description])
                                ortholog_group = 'None'
                                unknown_counter+= 1
                                TOTAL_COUNT += 1
                                TOTAL_UNKNOWN_COUNT += 1
                except category_found:
                    TOTAL_COUNT += 1
                # BLASTHIT = [id, accession_no, species, fasta_description, ortholog group (only if determined with high probability)]
                # id is a string (used to be an integer)!
                BLASTHIT = [hit_id, hit_accession_no, species, hit.description, ortholog_group]
                add_to_protein_hitdict(taxon, protein, hit_id, hit.description, ortholog_group, hit_accession_no)
                with conn:
                    # If the id is already in the database, the insertion fails and 0 is returned (otherwise the id is returned)
                    print('\n\nINSERT PROTEIN:\n{0}\n\n'.format(BLASTHIT))
                    print(str(db_insert_protein(BLASTHIT, True))+'\n')
                i += 1
            # Write ortholog, paralog and unknown lists individually for each protein/taxon
            if len(list_to_scrutinize_ortholog) > 0:
                write_to_html_scrutinize_file(protein, taxon, list_to_scrutinize_ortholog)
            else:
                print('No orthologs to write to HTML file.')
            if len(list_to_scrutinize_paralog) > 0:
                write_to_html_scrutinize_file(protein, taxon, list_to_scrutinize_paralog)
            else:
                print('No paralogs to write to HTML file.')
            if len(list_to_scrutinize_manually_excluded) > 0:
                write_to_html_scrutinize_file(protein, taxon, list_to_scrutinize_manually_excluded)
            else:
                print('No manually excluded proteins to write to HTML file.')
            if len(list_to_scrutinize_unknown) > 0:
                write_to_html_scrutinize_file(protein, taxon, list_to_scrutinize_unknown)
            else:
                print('No unknown proteins to write to HTML file.')
            print('Analysis for {0}/{1} completed.\n'.format(taxon, protein))
            print('Number of true-positives:'.format())
            control_counter = 0
            for key, value in positive_dict.items():
                print('                         '+key, str(value))
                control_counter += value
            print('Number of false-positives:'.format())
            for key, value in negative_dict.items():
                print('                         '+key, str(value))
                control_counter += value
            print('Number of unidentified homologous proteins: {0}\n'.format(unknown_counter))
            control_counter += unknown_counter
            print('Number of excluded proteins: {0}\n'.format(excluded_counter))
            control_counter += excluded_counter
            print('{0} out of {1} hits were analyzed and {2} could be assigned into categories.\n\n'.format(i, number, control_counter))
            new_protein_data[protein] = [number, negative_dict, positive_dict]
        print('Analysis for all proteins in taxon {0} completed.\n'.format(taxon))
        print('new_protein_data: {0}'.format(new_protein_data))

        # Returns a dictionary with related proteins and the number of hits for them (according to fasta description)
        print('\nRETURNED PROTEIN_DATA FOR TAXON {0}:\n'.format(taxon))
        for protein, value in new_protein_data.items():
            print('{0}:\n{1}'.format(protein, value))
        return new_protein_data

# This function should return the most common string in the hit description list
def run_backcheck_blast(ID, proteindata):
    # We need to define that this function should use the global variable LAST_BLAST_REPLY_TIME,
    # because we are changing LAST_BLAST_REPLY_TIME inside the function!
    global LAST_BLAST_REPLY_TIME
    try:
        blastp_result = SearchIO.read('data/backcheck/{0}.xml'.format(ID), 'blast-xml')
    except Exception as ex:
        print('No local backcheck blast results for accession no {0}. Need to run remote blast.'.format(ID))
        success = False
        try:
            # Make maximally 5 tries for any protein with increasing waiting time
            i = 0
            while success == False and i < 5:
                # Sleep if the last blast request was answered less than BLAST_WAITING_TIME (default = 60) seconds ago
                # This is necessary in order not to overload the server and become blocked.
                SECONDS_SINCE_LAST_BLAST = time.time()-LAST_BLAST_REPLY_TIME
                if SECONDS_SINCE_LAST_BLAST < BLAST_WAITING_TIME*(i+1):
                    print('Waiting for {0} seconds before initiating new blast request...'.format(round(BLAST_WAITING_TIME*(i+1)-SECONDS_SINCE_LAST_BLAST)))
                    time.sleep(BLAST_WAITING_TIME*(i+1)-SECONDS_SINCE_LAST_BLAST)
                else:
                    print('{0} seconds since last blast result was received. Continue without waiting...'.format(round(SECONDS_SINCE_LAST_BLAST)))
                    # remove this after checking
                    time.sleep(2)
                blast_start_time = time.time()
                print('Executing backcheck blast for accession no {0}. Waiting for results...'.format(ID))
                # Maybe hitlist-Size should be increased to 100 and expect to 10?
                result_handle = NCBIWWW.qblast("blastp", "nr", ID , hitlist_size = 50, expect = 0.01, format_type = "XML")
                i += 1
                with open('data/backcheck/{0}.xml'.format(ID), 'w') as out_handle:
                    out_handle.write(result_handle.read())
                result_handle.close()
                # Read in the data again to check that they contain really blast results
                with open('data/backcheck/{0}.xml'.format(ID), 'r') as xml_file:
                    xml_string = xml_file.read()
                    # This print command will print lots of lines...
                    #print('data:\n{0}'.format(data))
                    # The database was not searched. Sometimes there is no other error message,
                    # but all query results contain this string if there is no result
                    if '<Statistics_db-num>0</Statistics_db-num>' not in xml_string:
                        success = True
                        print('The blast search Was successful. Continuing...')
                    else:
                        print('No sequences were returned from blast server, repeating query...')
                xml_file.close()
                LAST_BLAST_REPLY_TIME = time.time()
                print('The remote blasting took {0}.'.format(execution_time_str(LAST_BLAST_REPLY_TIME-blast_start_time)))
        except Exception as ex:
            # Sequence id not found error
            if 'Message ID#24 Error' in str(ex):
                print("The blast server did not know the accession number. Error: " + str(ex))
                return 'unknown'
            else:
                print("Something went wrong with the backcheck blast. Perhaps the Blast server did not respond? More about the error: " + str(ex))
    else:
        print('Using local backcheck blast results (data/backcheck/{0}.xml).'.format(ID))
    blastp_result = SearchIO.read('data/backcheck/{0}.xml'.format(ID), 'blast-xml')
    how_many = len(blastp_result)
    print('blastp result should equal 50: {0}'.format(how_many))
    i = 0 # count synonyms
    j = 0 # count related proteins
    k = 1 # count 50 rounds
    for hit in blastp_result:
        if k == 1:
            print('Checking blast result {0}'.format(k), end=' ')
        else:
            print('{0}'.format(k), end = ' ')
        for synonym in proteindata[3]:
            if synonym.lower() in str(hit.description).lower():
                i += 1
                break
        for related_protein in proteindata[4]:
            if related_protein.lower() in str(hit.description).lower():
                j += 1
                break
        k += 1
    print('\n{0} from {1} backblast results confirm that {2} is {3}'.format(i, how_many, ID, proteindata[3][0]))
    print('{0} from {1} backblast results confirm that {2} is a related protein'.format(j, how_many, ID))
    # THRESHOLD needs to be adjusted after manually scrutinizing the results- Maybe 0.3 is sufficient!
    THRESHOLD = 0.5
    if how_many > 0:
        if i/how_many >= THRESHOLD:
            return 'synonym'
        elif j/how_many >= THRESHOLD:
            return 'related_protein'
        else:
            return 'unknown'
    else:
        return 'unknown'

def get_fully_sequenced_genomes(CSV_FILE, PHYLUM_SPECIES_FILE):
    print('------------------------------------------------ ---------')
    print('--------\nGET NUMBERS OF FULLY SEQUENCED GENOMES\n--------')
    print('------------------------------------------------- --------')
    # Open/download the list of fully sequenced genomes
    try:
        input_csv_file = csv.DictReader(open(CSV_FILE))
    except FileNotFoundError as err:
        print(err)
        # Download list of all fully sequences animal genomes:
        URL = 'https://www.ncbi.nlm.nih.gov/genomes/solr2txt.cgi?q=%5Bdisplay()%5D.from(GenomeBrowser).usingschema(%2Fschema%2FGenomeAssemblies).matching(group%3D%3D%5B%22Animals%22%5D)&fields=organism%7COrganism%20Name%2Clineage%7COrganism%20Groups%2Csize%7CSize(Mb)%2Cchromosomes%7CChromosomes%2Corganelles%7COrganelles%2Cplasmids%7CPlasmids%2Cassemblies%7CAssemblies&filename=genomes.csv&nolimit=on'
        r = requests.get(URL)
        with open('CSV_FILE', 'wb') as file:
            file.write(r.content)
        input_csv_file = csv.DictReader(open(CSV_FILE))
    # Initialize dictionary with zeros
    full_genome_dictionary = {}
    phylum_species_dictionary = {'unknown': ''}
    print('taxon_dictionary_print:\n{0}'.format(taxon_dictionary))
    for taxon, taxon_data in taxon_dictionary.items():
        full_genome_dictionary[taxon] = 0
        print('Adding {0} to full_genome_dictionary.'.format(taxon))
        phylum_species_dictionary[taxon] = ''
        print('Adding {0} to phylum_species_dictionary.'.format(taxon))
    for line in input_csv_file:
        species_name = line['#Organism Name']
        number_of_assemblies = line['Assemblies']
        print('species_name = {0}'.format(species_name))
        TAXON_ID, phylum = get_taxon_id_and_phylum(species_name)
        print('TAXON_ID: {0}, phylum: {1}'.format(TAXON_ID, phylum))
        # If phylum cannot be resolved via NBI, this would give an error when trying to increment the
        # number of fully sequenced genomes! Count only genomes as "fully sequenced" if at least one
        # genome assembly exists (the file format has three numbers for the assemblies in the end of the rows
        # and the first number needs to be a 1 or bigger; hence >99 for fully sequenced genomes. The other
        # numbers refer to mitochondrial and other plasmid DNA if I am not mistaken)
        if int(number_of_assemblies) > 99:
            if phylum != 'unknown':
                full_genome_dictionary[phylum] += 1
                phylum_species_dictionary[phylum] += '{0}, '.format(species_name)
            else:
                phylum_species_dictionary['unknown'] += '{0}, '.format(species_name)
            print('Adding +1 for {0} ({1} assemblies) to fully sequenced {2}'.format(species_name, number_of_assemblies, phylum))
        else:
            print('Number of assemblies for {0}: {1}'.format(species_name, number_of_assemblies))
    print(str(full_genome_dictionary))
    # Write phylum/species list to file
    with open(PHYLUM_SPECIES_FILE, 'w') as file:
        for phylum, species in phylum_species_dictionary.items():
            file.write('{0}: {1}\n'.format(phylum, species))
    return full_genome_dictionary

def create_connection(DATABASE_FILE):
    global conn
    print('Trying to connect to database file {0}...'.format(DATABASE_FILE))
    if os.path.isfile(DATABASE_FILE) and os.stat(DATABASE_FILE).st_size != 0:
        try:
            conn = sqlite3.connect(DATABASE_FILE)
            return conn
        except Exception as err:
            print('Database file {0} not found.'.format(DATABASE_FILE))
            print('Error: {0}'.format(err))
    else:
        print('Database file {0} does not exist or is empty. Trying to create a new one.'.format(DATABASE_FILE))
        if create_sqlite_file(DATABASE_FILE):
            conn = sqlite3.connect(DATABASE_FILE)
            return conn
        else:
            print('Database file {0} not found and unable to create a new one.'.format(DATABASE_FILE))

def db_insert_protein(BLASTHIT, VERBOSE = True):
    global conn
    # Check whether an entry exists already
    if VERBOSE: print('Checking whether an entry with id {0} exists already in the SQLite database... '.format(BLASTHIT[0]), end='')
    #for key in sqlite_protein_dict:
    #    print(key, end=' ')
    if BLASTHIT[0] in sqlite_protein_dict.keys():
        print('Yes')
        # Check whether the entry has already the ortholog group set
        print('ortholog group: {0}'.format(sqlite_protein_dict[BLASTHIT[0]]))
        # Only try to update if the new ortholog_group is not 'None' and the old one is 'None'
        if BLASTHIT[4] != 'None' and sqlite_protein_dict[BLASTHIT[0]][4] == 'None':
            # Update database entry to include ortholog group
            if VERBOSE: print('Entry with id = {0} has no ortholog group information. Trying to update with \"{1}\".'.format(BLASTHIT[0], BLASTHIT[4]))
            query = 'UPDATE protein SET ortholog_group = \'{0}\' WHERE id = \'{1}\''.format(BLASTHIT[4], BLASTHIT[0])
            if VERBOSE: print('query: {0}'.format(query))
            try:
                cur = conn.cursor()
                cur.execute(query)
                result = cur.lastrowid
                conn.commit()
            except sqlite3.Error as err:
                if VERBOSE: print('Database error during update: {0}'.format(err))
                result = 0
            except Exception as err:
                if VERBOSE: print('Unknown error during update: {0}'.format(err))
                result = 0
            else:
                print('Ortholog group of id {0} updated to {1}.'.format(BLASTHIT[0], BLASTHIT[4]))
        else:
            print('Ortholog group is already set ({0}).'.format(BLASTHIT[4]))
            result = 0
    else:
        print('No')
        if VERBOSE: print('No entry with id = {0}. Trying to insert.'.format(BLASTHIT[0]))
        # This is a full database insertion
        # In some fasta descriptions one can really find this char: ' !!!!
        BLASTHIT[2] = BLASTHIT[2].replace('\'', '')
        query = 'INSERT INTO protein (id, accession_no, species, fasta_description, ortholog_group) VALUES (\'{0}\', \'{1}\', \'{2}\', \'{3}\', \'{4}\')'.format(BLASTHIT[0], BLASTHIT[1], BLASTHIT[2], BLASTHIT[3], BLASTHIT[4])
        if VERBOSE: print('query: {0}'.format(query))
        try:
            cur = conn.cursor()
            cur.execute(query)
            result = cur.lastrowid
            conn.commit()
        except sqlite3.Error as err:
            if VERBOSE: print('Database error during insertion: {0}'.format(err))
            result = 0
        except Exception as err:
            if VERBOSE: print('Unknown error during insertion: {0}'.format(err))
            result = 0
    return result

# Give a list of scientific_name, taxon_id and phylum to insert into SQLite database
# There should never be the need to update any entry from this table, only to insert
def db_insert_species(SPECIES_NAME, TAXON_ID, PHYLUM, VERBOSE=True):
    global conn
    if VERBOSE: print('Trying to execute SQL insertion ({0})...'.format(SPECIES_NAME))
    query = 'INSERT INTO species (scientific_name, taxon_id, phylum) VALUES (\'{0}\', {1}, \'{2}\')'.format(SPECIES_NAME, TAXON_ID, PHYLUM)
    if VERBOSE: print('query: {0}'.format(query))
    try:
        cur = conn.cursor()
        cur.execute(query)
        result = cur.lastrowid
        if VERBOSE: print('SQL insertion successful.')
        conn.commit()
    except sqlite3.Error as err:
        if VERBOSE: print('Database error: {0}'.format(err))
        result = 0
    except Exception as err:
        if VERBOSE: print('Unknown error: {0}'.format(err))
        result = 0
    return result

# Return a list of scientific_name, taxon_id and phylum when queried by scientific_name
def db_retrieve_species(SPECIES_NAME, VERBOSE=True):
    global conn
    if VERBOSE: print('Trying to retrieve SQLite data for {0}...'.format(SPECIES_NAME))
    query = 'SELECT * FROM species WHERE scientific_name = \'{0}\''.format(SPECIES_NAME)
    if VERBOSE: print('query: {0}'.format(query))
    try:
        cur = conn.cursor()
        cur.execute(query)
        result = cur.fetchone()
        #print('\nRESULT: {0}\n'.format(result))
    except sqlite3.Error as err:
        if VERBOSE: print('Database error: {0}'.format(err))
        result = 0
    except Exception as err:
        if VERBOSE: print('Unknown error: {0}'.format(err))
        result = 0
    # There should be only one row!
    return result

def load_sqlite_table_to_dict(TABLENAME, VERBOSE = True):
    global conn
    if VERBOSE: print('\n\nLoading table {0} from SQLite database...'.format(TABLENAME))
    query = 'SELECT * FROM {0}'.format(TABLENAME)
    if VERBOSE: print('query: {0}'.format(query))
    try:
        cur = conn.cursor()
        cur.execute(query)
        results = cur.fetchall()
    except sqlite3.Error as err:
        # No or empty database file
        if VERBOSE: print('Database error: {0}'.format(err))
    except Exception as err:
        if VERBOSE: print('Unknown error: {0}'.format(err))
    else:
        sqlite_dict = {}
        i = 0
        for row in results:
            sqlite_dict[row[0]] = row
            i += 1
        print('Loading of table {0} from SQLite database was successful ({1} entries):\n'.format(TABLENAME, i))
        print('table {0}:\n{1}\n\n'.format(TABLENAME, sqlite_dict))
    return sqlite_dict

def run():
    global APPLICATION_PATH, taxon_dictionary, master_dictionary, synonym_dictionary, blacklist, DATABASE_FILE, LAST_BLAST_REPLY_TIME, BLAST_WAITING_TIME, TOTAL_COUNT, TOTAL_UNKNOWN_COUNT, conn, sqlite_protein_dict, sqlite_species_dict, protein_hitdict, excluded_list, TRUE_UNIQUES, SVG_DICT
    TRUE_UNIQUES = {}
    TOTAL_COUNT = 0
    TOTAL_UNKNOWN_COUNT = 0
    # You can adjust this down until you see that the blat server starts blocking your requests!
    # 60 (seconds) is a very conservative (but slow) estimate, that does not result in blocking
    BLAST_WAITING_TIME = 30
    LAST_BLAST_REPLY_TIME = time.time()-BLAST_WAITING_TIME
    print(LAST_BLAST_REPLY_TIME)
    # Determine directory of script (in order to load the data files)
    APPLICATION_PATH =  os.path.abspath(os.path.dirname(__file__))
    #print('\nThe script is located in {0}'.format(APPLICATION_PATH))
    TAXON_DICTIONARY_FILE = '{0}/data/taxon_data.py'.format(APPLICATION_PATH)
    CSV_FILE = '{0}/data/genomes.csv'.format(APPLICATION_PATH)
    PHYLUM_SPECIES_FILE = '{0}/data/phylum-sorted_list_of_fully_sequenced_genomes.txt'.format(APPLICATION_PATH)
    DATABASE_FILE = '{0}/data/database.sqlite3'.format(APPLICATION_PATH)
    LOGFILE = '{0}/data/logfile.txt'.format(APPLICATION_PATH)
    ANALYSIS_RESULTS_DIR = '{0}/data/analysis_results'.format(APPLICATION_PATH)
    PROTEIN_RESULTS_DIR = '{0}/data/protein_results'.format(APPLICATION_PATH)
    # Remove all old analysis and protein results and recreate the directories
    if os.path.isdir(ANALYSIS_RESULTS_DIR):
        shutil.rmtree(ANALYSIS_RESULTS_DIR)
        os.mkdir(ANALYSIS_RESULTS_DIR)
    else:
        os.mkdir(ANALYSIS_RESULTS_DIR)
    if os.path.isdir(PROTEIN_RESULTS_DIR):
        shutil.rmtree(PROTEIN_RESULTS_DIR)
        os.mkdir(PROTEIN_RESULTS_DIR)
    else:
        os.mkdir(PROTEIN_RESULTS_DIR)
    preamble1, taxon_dictionary = load_dictionary(TAXON_DICTIONARY_FILE)
    #print('taxon_dictionary: {0}\n'.format(taxon_dictionary))
    #print("Populating new taxon dictionary")
    new_taxon_dictionary = taxon_dictionary
    preamble2, master_dictionary = load_dictionary('{0}/data/master_dictionary.py'.format(APPLICATION_PATH))
    # Make a list of all manually identified false-positive hits
    excluded_list = os.listdir('{0}/data/proteins_exclude'.format(APPLICATION_PATH))
    print('Excluded list:\{0}'.format(excluded_list))
    # Downlaod reference proteins and rename fasta description according to the key (human-readable name)
    download_proteins('{0}/data/reference_proteins'.format(APPLICATION_PATH), master_dictionary, rename_fasta_description_after_key=True, overwrite=False, filename='after_value[0]')
    synonym_dictionary = make_synonym_dictionary(master_dictionary)
    print('synonym_dictionary:\n{0}'.format(synonym_dictionary))
    blacklist = load_blacklist()
    # {hit_accession_no: [taxon, protein, hit_description, ortholog_group]}
    protein_hitdict = {}
    conn = create_connection(DATABASE_FILE)
    print('{0}'.format(conn))
    sqlite_protein_dict = load_sqlite_table_to_dict('protein', VERBOSE = True)
    sqlite_species_dict = load_sqlite_table_to_dict('species', VERBOSE = True)
    SVG_DICT = {}
    for taxon, taxon_data in taxon_dictionary.items():
        SVG_DICT[taxon] = {}
        # populate the field with "0"
        TRUE_UNIQUES[taxon] = 0
        print('949 After initializing: TRUE_UNIQUES[{0}]: {1}'.format(taxon, TRUE_UNIQUES[taxon]))
        #print("Enter recursion")
        if taxon not in blacklist:
            #print("Passed blacklist loading")
            # Replace old data if new data is available
            # PROTEIN DATA (how many hits, false-positive list)
            try:
                new_protein_data = get_protein_data(taxon)
            except Exception as err:
                print('Error was: {0}'.format(err))
            else:
                new_taxon_dictionary[taxon][4] = new_protein_data
            # TAXON_ID
            # Get taxon_id if a new taxon was added manually to taxon_data.py, but no taxon_id was given
            # This is not necessary if the taxon_id is always manually assigned when adding a new taxon
            # and it also fails when adding a complex taxon
            #try:
            #    new_taxon_id, phylum = get_taxon_id_and_phylum(taxon)
            #except Exception as err:
            #    print('Could not get new taxon id and phylum. Error was: {0}',value(err))
            # execute if no exceptions have occured
            #else:
            #    new_taxon_dictionary[taxon][0] = new_taxon_id
            #
            # NUMBER OF SPECIES IN NCBI DATABASE
            try:
                new_species_number = get_species_number_from_ncbi(taxon, taxon_data)
            except Exception as err:
                print('Could not get new species number. Error was: {0}',value(err))
            else:
                new_taxon_dictionary[taxon][1] = new_species_number
            # NUMBER OF SEQUENCES IN NCBI DATABASE
            try:
                new_sequence_number = get_sequence_number(taxon, taxon_data)
            except Exception as err:
                print('Could not get new sequence number. Error was: {0}',value(err))
            else:
                new_taxon_dictionary[taxon][3] = new_sequence_number
            # Wait in order not to overload the server
            time.sleep(1)
    # Add the number of fully sequenced genomes to the new taxon dictionary
    full_genome_dictionary = get_fully_sequenced_genomes(CSV_FILE, PHYLUM_SPECIES_FILE)
    for taxon, number in full_genome_dictionary.items():
        new_taxon_dictionary[taxon][5] = number
    conn.commit()
    # Count the total number of unique homologs for each taxon and insert it into taxon_dictionary,
    # also count total number of unique homologs
    total_number_of_unique_homologs = 0
    for taxon in taxon_dictionary:
        all_uniques = sum(value[0] == taxon for value in protein_hitdict.values())
        print('Sum for {0}: {1}'.format(taxon, all_uniques))
        new_taxon_dictionary[taxon][6] = all_uniques
        total_number_of_unique_homologs += all_uniques
    #print('\nWriting new_taxon_dictionary:\n{0}'.format(str(new_taxon_dictionary)))
    # Make backup file before overwriting
    os.rename(TAXON_DICTIONARY_FILE, TAXON_DICTIONARY_FILE+'.bak')
    # This used to be after line 1016
    write_protein_hitdict_to_file(protein_hitdict)
    # Add the real number of homologs (considering the manually excluded sequences)
    for taxon in taxon_dictionary:
        try:
            print('TRUE_UNIQUES for taxon {0}: {1}'.format(taxon, TRUE_UNIQUES))
            new_taxon_dictionary[taxon][7] = TRUE_UNIQUES[taxon]
        except Exception as err:
            print('Error: TRUE_UNIQUES for taxon {0}: {1}; err: {2}'.format(taxon, TRUE_UNIQUES, err))
            # The TRUE_UNIQUES number is only calculated until LIMIT['MAX'], which is by default 200)
            new_taxon_dictionary[taxon][7] = '-'
    # Write new taxon data to file
    write_dict_to_file(preamble1, new_taxon_dictionary, TAXON_DICTIONARY_FILE)

    # Format the taxon_data.py file
    if insert_line_breaks(TAXON_DICTIONARY_FILE) == True:
        print('Successfully formatted the taxon data file.')
    else:
        print('Formating the taxon data file failed.')
    number_of_blast_searches = len(taxon_dictionary)*len(master_dictionary)
    save_stats = 'Analyzed sequences (hits resulting from {0} blast searches, {1} animal groups x {2} query sequences):'.format(number_of_blast_searches, len(taxon_dictionary), len(master_dictionary))
    save_stats += '{0} (out of which unique: {1}, programmatically recognized as VEGF/PDGF family members: {2}%)\n'.format(TOTAL_COUNT, total_number_of_unique_homologs, round(100-TOTAL_UNKNOWN_COUNT*100/TOTAL_COUNT, 1))
    with open(LOGFILE, 'a') as log_file:
        log_file.write(save_stats)
    print(save_stats)
    # Delete all guide trees in tree directory
    print('Deleting all guide tree files.')
    treedir = '{0}/data/alignments_and_trees/'.format(APPLICATION_PATH)
    for file in os.listdir(treedir):
        if file.endswith('_all.dnd'):
            os.remove('{0}/{1}'.format(treedir, file))

if __name__ == '__main__':
    run()


import sqlite3
# from flask import jsonify
# from flask import request

conn = sqlite3.connect('cards.db')
c = conn.cursor()

# c.execute("""
#     CREATE TABLE DominionGames (
#     game_id integer PRIMARY KEY,
#     Player1Name text NOT NULL,
#     Player1Score text NOT NULL,
#     Player2Name text NOT NULL,
#     Player2Score text NOT NULL,
#     Player3Name text,
#     Player3Score text,
#     Player4Name text,
#     Player4Score text)""")
# print("Table made succsessfully")

#c.execute("INSERT INTO DominionGames (game_id, Player1Name, Player1Score, Player2Name, Player2Score) VALUES ('1', 'Josh', '10', 'John', '8')")

c.execute("SELECT * FROM DominionGames")
print(c.fetchall())


import numpy as np
import matplotlib.pyplot as plt

def f(x):
    if x < 0 or x > 1:
        return 0
    else:
        return 1 - x

def g(x):
    if x < 0 or x > 2:
        return 0
    else:
        return 1 - x/2

def riemann(f, g, bounds = [-3, 4], delta_x = 0.001):
    s = 0
    for x in np.arange(bounds[0], bounds[1], delta_x):
        s += f(x) * g(x) * delta_x
    return s

def convo(f, g, delta_u = 0.001):
    h = []
    for u in np.arange(-5, 5, delta_u):
        print(u)
        h.append(riemann(f, lambda y: g(u - y)))

    return h

h = convo(f, g)

plt.plot(np.arange(-5, 5, 0.001), h)
plt.show()


import sqlite3

'''LETS KNOW THE DATA TYPES WE HAVE IN sqlite3
Null  --> means doesnot exist
integers --> means real numbers
REAL --> means numbers with decimal
TEXT  --> means just a Text
BLOB  --> means kind of images mp3file or music file '''

'''STEPS IN CREATING A DATA IN THE DATABASE'''
# #STEP ONE:
# # creating a connection

# # con = sqlite3.connect('customer.db')  #when you run this program like this it is going to create a db file\

# #STEP ONE:
# # creating a connection
# '''creating a table and adding table to the table'''
# conn = sqlite3.connect('customer.db')
# # first we need to create a cursor
# # cursor points what we want to do with our table

# #STEP TWO:
# # creating a cusor
# c = conn.cursor()

# #STEP THREE:
# # creata a table
# #when we want to create a table we use our cursor (command of what our cosor will do)
# #sqlite3 IS CASE SENSITIVE SO  FOLLOW THE SAME STEP Like UPPERCASE CREATE TABLE and lowercase customer

# '''using a single quotaion mark can make ur code not to be readable'''

# # c.execute("CREATE TABLE customers(  first_name DATATYPE, last_name DATATYPE, email_address DATATYPE) ")

'''using the three quotation mark here make ur code readale'''

# c.execute("""CREATE TABLE customers(
#         first_name text,
#         last_name text,
#         email_address text

# ) """)

# #STEP FOUR:
# # COMMIT OUR COMMAND TO THE DATABASE(push what ever we have to the database)

# conn.commit()

#STEP FIVE:
# # close our  connection (anytime we create a connection we should always close it best practice)
# conn.close()

'''INSERT ONE RECORD TO THE DATABASE'''
conn = sqlite3.connect('customer.db')

c = conn.cursor()

# # EXECUTE OUR cusor
c.execute("INSERT INTO customers VALUES ('Mary', 'Doe', 'mary@gmail.com') ")  #sqlite3 COMMAND TO INSERT TABLE

print('command executed successfully...')
conn.commit()
conn.close()

'''INSERT MANY  TABLE TO THE DATABASE'''

# conn = sqlite3.connect('customer.db')

# c = conn.cursor()

# many_customers = [('chuks', 'emeka', 'chuls@gmail.com'),
#                  ('emy', 'joe', 'joe@gmail.com'),
#                  ('ebuka', 'elvin', 'e@gmail.com')]
# # EXECUTE OUR cusor
# c.executemany("INSERT INTO customers VALUES (?,?,?)", many_customers )  # ??? are the place holders for ech field

# print('command executed successfully...')
# conn.commit()
# conn.close()

'''QUERY AND FETCH ALL IN THE DATABASE'''

conn = sqlite3.connect('customer.db')

c = conn.cursor()

'''QUERY THE DATABASE'''
# c.execute("SELECT * FROM customers")
'''Primary ID'''
# c.execute("SELECT rowid, * FROM customers")

'''GET THE DATAS'''
# print(c.fetchone()[1])
# c.fetchmany(3)
# items = c.fetchall()
# for item in items:
#     print(item)
# EXECUTE OUR cusor

'''TO SEARCH AND FETCH USING THE WHERE CLAUSE'''

# c.execute("SELECT  * FROM customers WHERE last_name = 'Doe' ")
# c.execute("SELECT  * FROM customers WHERE email_address LIKE 'c%' ")
# item = c.fetchall()
# print(item)

'''UPDATE RECORD'''
# c.execute("""UPDATE customers SET first_name = 'ezinne'
#           WHERE last_name = 'emeka'

#           """)
'''a better way  todo this is to use a roll id'''
# c.execute("""UPDATE customers SET first_name = 'Chisom'
#           WHERE rowid = '5'

#           """)

'''DELETE RECORD'''
# c.execute("DELETE from customers WHERE rowid = 6")

'''we need to commit this'''
# conn.commit()

# c.execute("SELECT rowid, * FROM customers")

'''QUERY THE DATABASE ORDER BY'''
#by accending
c.execute("SELECT rowid, * FROM customers ORDER BY rowid")
# #by decending
# c.execute("SELECT rowid, * FROM customers ORDER BY rowid DESC")  #-rowid
# #by lastname with assending
# c.execute("SELECT rowid, * FROM customers ORDER BY last_name ")

# #by lastname with decending
# c.execute("SELECT rowid, * FROM customers ORDER BY last_name DESC")

'''QUERY THE DATABASE ORDER BY AND and OR'''
# c.execute("SELECT rowid, * FROM customers WHERE last_name LIKE 'D%' AND rowid = 2 ")
# c.execute("SELECT rowid, * FROM customers WHERE last_name LIKE 'D%' OR rowid = 2 ")

'''LIMITING RESULTS'''
# c.execute("SELECT rowid, * FROM customers  ")

'''CREATING A FUNCTION TO USE IN OUR SIMPLEAPP.PY'''
# c.execute("SELECT rowid, * FROM customers ORDER BY rowid LIMIT 1")

# item = c.fetchall()
# # for i in item:
# #     print(i)

# print(item)

# # print('command executed successfully...')
# '''every database created must end with commit and close'''
# conn.commit()
# conn.close()

'''CREATING A FUNCTION TO USE IN OUR SIMPLEAPP.PY'''
# query the database and return all records
def show_all():

    #connecting to the database
    conn = sqlite3.connect('customer.db')

    #creating a cursor
    c = conn.cursor()
    # query the database
    c.execute("SELECT rowid, * FROM customers ")

    item = c.fetchall()
    for i in item:
        print(i)

    # print('command executed successfully...')
    '''every database created must end with commit and close'''
    conn.commit()
    conn.close()

'''ADD A NEW RECORD TO OUR TABLE'''
def add_one(first_name, last_name, email_address):
     #connecting to the database
    # conn = sqlite3.connect('customer.db')
    c = conn.cursor()
    # executing and passing in the  parameters
    c.execute("INSERT INTO customers VALUES (?,?,?)", (first_name, last_name, email_address))
    conn.commit()
    conn.close()

'''ADD A NEW MANY TO OUR TABLE'''
def add_many(list):

    conn = sqlite3.connect('customer.db')
    c = conn.cursor()
    c.executemany("INSERT INTO customers VALUES (?,?,?)", (list))

    conn.commit()
    conn.close()

# 1:24:45


# -*- coding: utf-8 -*-
"""
script launched during summer with Patricia coming
"""
import cv2
import datetime
import mutex
import numpy as np
import os
import time
import threading

#~ from skimage.measure import structural_similarity as ssim # apt-get install python-skimage
#~ import paramiko

# Captures a single image from the camera and returns it in PIL format
def get_image(camera):
     # read is the easiest way to get a full image out of a VideoCapture object.
     retval, im = camera.read()
     return im

def is_available_resolution(cam,x,y):
    cam.set(cv2.cv.CV_CAP_PROP_FRAME_WIDTH, int(x))
    cam.set(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT, int(y))
    return cam.get(cv2.cv.CV_CAP_PROP_FRAME_WIDTH) == int(x) and cam.get(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT) == int(y)

def get_webcam_available_resolution(camera):
    aTestRes = [
                            160,120,
                            320,240,
                            640,480,
                            800,600,
                            1024,768,
                            1280,960,
                            1280,1024,
                            1600,1200,
                            1920,1080, # HD
                            1920,1440,
                            2048,1280,
                            2560,1080,
                            4096,2304,
                            ]

    aRes = []
    i = 0
    while i < len(aTestRes ):
        if is_available_resolution( camera, aTestRes[i], aTestRes[i+1] ):
            aRes.append( [ aTestRes[i], aTestRes[i+1] ] )
        i += 2

    # after testing, set standard camera resolution
    camera.set(cv2.cv.CV_CAP_PROP_FRAME_WIDTH,640)
    camera.set(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT,480)

    return aRes
# get_webcam_available_resolution - end

def find_nearest_resolution( aAvailableResolution, nApproximativeWidth ):
    """
    find nearest resolution
    """
    nMinDiff = 999999
    resBest = []
    for res in aAvailableResolution:
        nDiff = abs( res[0] - nApproximativeWidth )
        if nMinDiff > nDiff:
            nMinDiff = nDiff
            resBest = res
    return resBest
# find_nearest_resolution - end

def computeDiff( im1, im2 ):
    """
    compute difference between two images.
    return nbr_different_pixels, ratio compared to nbr pixel in images
    """
    #~ print( "im1: %s" % im1[8][:16] )
    #~ print( "im2: %s" % im2[8][:16] )
    diff = abs(im1.astype("int16") - im2.astype("int16"))

    print( "diff: %s" % diff[8][:16] )
    ret,thresh1 = cv2.threshold( diff, 32, 1, cv2.THRESH_BINARY ) # nbr of different pixel
    #~ print( "thresh1: %s" % thresh1[8][:16] )
    #~ retVal = cv2.mean(thresh1)[0] # 0.13 for HD on Raspv2
    retVal = cv2.countNonZero(thresh1)
    retVal = ( int(retVal),float(retVal)/( im1.shape[0]*im1.shape[1] ) )
    print( "retVal: %s" % str(retVal) )
    return retVal

def mseFloat(imageA, imageB):
	# the 'Mean Squared Error' between the two images is the
	# sum of the squared difference between the two images;
	# NOTE: the two images must have the same dimension
	err = np.sum( ((imageA.astype("float") - imageB.astype("float")) ** 2) )
	err /= float(imageA.shape[0] * imageA.shape[1])

	# return the MSE, the lower the error, the more "similar"
	# the two images are
	return err

def mse(imageA, imageB):
	# the 'Mean Squared Error' between the two images is the
	# sum of the squared difference between the two images;
	# NOTE: the two images must have the same dimension
	err = np.sum( ((imageA.astype("int16") - imageB.astype("int16")) ** 2) ) # astype("float"): 0.28s in HD astype("int"): 0.15s astype("int16"): 0.11s
	err /= float(imageA.shape[0] * imageA.shape[1])

	# return the MSE, the lower the error, the more "similar"
	# the two images are
	return abs(err)

def sendFileToRemoteParamiko( server, user, srcfilename, dstfilename ):
    import paramiko
    def createSSHClient(server, port, user, password):

        client = paramiko.SSHClient()
        client.load_system_host_keys()
        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        client.connect(server, port, user, password)
        return client
    ssh = createSSHClient("protolab.aldebaran.com", 22, "", "")
    scp = SCPClient(ssh.get_transport())
    #~ Then call scp.get() or scp.put() to do scp operations.
    # TODO: finish me !

def sendFileToRemote( server, user, srcfilename, dstfilename ):
    strCommand = "scp %s %s@%s:%s" % (srcfilename, user, server,  dstfilename) # -C for large image could be usefull
    print( "INF: running command: '%s'" % strCommand )
    os.system( strCommand )

import subprocess

class BackgroundSender:
    """
    Permis to ask to send a bunch of files, but it send only one at a time.
    After the file is finished to send, it try to send the last(s) one
    """
    def __init__( self, servername, username, nQueueLen = 40, nNbrSimultaneousSend = 1 ):
        self.nQueueLen = nQueueLen
        self.nNbrSimultaneousSend = nNbrSimultaneousSend # NDEV
        self.servername = servername
        self.username = username

        self.listWaiting = [] # a list of pair src/dst file(s) to be send
        self.mutex = mutex.mutex()

        self.bst = None
        self.launchBackgroundThread()

    def launchBackgroundThread(self):
        class BackgroundSenderThread( threading.Thread ):
            def __init__(self, backgroundSender ):
                threading.Thread.__init__( self );
                self.backgroundSender = backgroundSender
            # init - end

            def run ( self ):
                print( "DBG: launchBackgroundThread.run: start" );

                while( 1 ):
                    while self.backgroundSender.mutex.testandset() == False:
                        print( "DBG: wait mutex in thread" )
                        time.sleep( 0.001 )

                    if len(self.backgroundSender.listWaiting) > 0:
                        # send always the last
                        srcfilename, dstfilename = self.backgroundSender.listWaiting[-1]
                        self.backgroundSender.listWaiting = self.backgroundSender.listWaiting[:-1]
                        print( "DBG: launchBackgroundThread.run: remaining in queue: %s" % len(self.backgroundSender.listWaiting) )
                        self.backgroundSender.mutex.unlock()

                        strCommandAndArgs =  "scp %s %s@%s:%s" % (srcfilename, self.backgroundSender.username, self.backgroundSender.servername,  dstfilename) # -C for large image could be usefull
                        print( "DBG: launchBackgroundThread.run: launching sub command: '%s'" % (strCommandAndArgs) )
                        newProcess = subprocess.Popen( strCommandAndArgs, shell=True ) # , stdin=subprocess.PIPE
                        try:
                            sts = os.waitpid( newProcess.pid, 0 )
                        except:
                            pass # pid already finished or some erros occurs or under windows ?
                        print( "DBG: %s => %s [OK]" % (srcfilename,dstfilename) )
                    else:
                        self.backgroundSender.mutex.unlock()
                        time.sleep( 0.4 )
                print( "DBG: launchBackgroundThread.run: stop" );
            # run - end
        # class BackgroundSenderThread - end
        self.bst = BackgroundSenderThread(self)
        self.bst.start()

    # launchBackgroundThread  - end

    def addFile( self, src, dst ):
        """
        Add a new transfer command
        """
        while self.mutex.testandset() == False:
            print( "DBG: wait mutex addFile" )
            time.sleep( 0.001 )
        print( "DBG: BackgroundSender.addFile: adding '%s'->'%s' task" % (src, dst) )
        if len(self.listWaiting) == self.nQueueLen:
            # clear older one
            self.listWaiting = self.listWaiting[1:] # drop
        self.listWaiting.append( (src, dst) )
        self.mutex.unlock()
# class BackgroundSender - end

def getFilenameFromTime(timestamp=None):
  """
  get a string usable as a filename relative to the current datetime stamp.
  eg: "2012_12_18-11h44m49s049ms"

  timestamp : time.time()
  """
  if timestamp is None:
      datetimeObject = datetime.datetime.now()
  elif isinstance(timestamp, datetime.datetime):
      datetimeObject = timestamp
  else:
      datetimeObject = datetime.datetime.fromtimestamp(timestamp)
  strTimeStamp = datetimeObject.strftime( "%Y_%m_%d-%Hh%Mm%Ss%fms" );
  strTimeStamp = strTimeStamp.replace( "000ms", "ms" ); # because there's no flags for milliseconds
  return strTimeStamp;
# getFilenameFromTime - end

def getHostName():
    "get the computer name as given by user"
    return os.uname()[1]
# getHostName - end

def launchAlarmSound():
    os.system( "aplay ~/salut_bureau.wav&" )

def watchAndSend( nNumCameraPort ):
    timeLastAlarmSound = time.time() - 100000
    #Number of frames to throw away while the camera adjusts to light levels
    ramp_frames = 1 # 30

    print("INF: Opening...")

    # Now we can initialize the camera capture object with the cv2.VideoCapture class.
    # All it needs is the index to a camera port.
    camera = cv2.VideoCapture(camera_port)

    availableResolution = get_webcam_available_resolution(camera)
    print( "availableResolution: %s" % availableResolution )
    #~ aResToUse = availableResolution[-1]
    aResToUse = find_nearest_resolution( availableResolution, 1280 )
    print( "using Resolution: %s" % aResToUse )

    camera.set(cv2.cv.CV_CAP_PROP_FRAME_WIDTH,aResToUse[0])
    camera.set(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT,aResToUse[1])

    print("INF: Warming up...")
    # Ramp the camera - these frames will be discarded and are only used to allow v4l2
    # to adjust light levels, if necessary
    for i in xrange(ramp_frames):
         temp = get_image(camera)
         #~ time.sleep( 0.05 )

    bStoreTmp = False
    strHostname = getHostName()
    strPrevDstFolderName = ""

    # NB: ssh-copy-id from this machine should have been done previously
    strRemoteHostName = "protolab.aldebaran.com"
    strRemoteUserName = "amazel"

    if not bStoreTmp:
        bgs = BackgroundSender( strRemoteHostName, strRemoteUserName )

    print("INF: Taking image for real...")

    imPrev = get_image(camera)
    imPrevGrey = cv2.cvtColor( imPrev, cv2.COLOR_BGR2GRAY )

    while( 1 ):
        #~ print( "DBG: %s: getting..." % ti    me.time() )
        timeBegin = time.time()
        im = get_image(camera) # it seems like camera got a buffer of 6 images, so even if taking time to send, we've got 6 images of moving datas in raw stored! great
        #~ print( "DBG: %5.02f: getting takes: (%5.2fs)" % ( time.time(), time.time()-timeBegin ) )

        #~ print( "DBG: %s: analysing..." % time.time() )
        timeBegin = time.time()
        imGrey = cv2.cvtColor( im, cv2.COLOR_BGR2GRAY )
        #~ print( "DBG: %5.02f: converting takes: (%5.2fs)" % ( time.time(), time.time()-timeBegin ) )

        timeBegin = time.time()
        #~ rDiff = computeDiff( imGrey, imPrevGrey )[1]
        rDiff = mse( imGrey, imPrevGrey )
        #~ rDiff = ssim(mGrey, imPrevGrey )
        #~ print( "DBG: %5.02f: rDiff: %5.1f (%5.2fs)" % ( time.time(), rDiff,time.time()-timeBegin ) )

        # mse: en640x480: 120 dans une piece, 50 sur une vue d'ensemble par la fenetre de la chambre
        # diff: en HD 170°: 0.009 en interieur quand rien de bouge (parfois 0.011)

        if rDiff > 150.012:
            timeBegin = time.time()
            print( "DBG: %s: storing..." % time.time() )
            if 0:
                dstFile = "~/images/%09.03f.jpg"  % time.time()
            else:
                dstFile = getFilenameFromTime() + ".jpg"
                # folder by hostname
                strDstFolderName = "~/images/" + strHostname + "/" + dstFile[:10] + "/" # one folder by day
                if strPrevDstFolderName != strDstFolderName:
                    os.system( "ssh %s@%s 'mkdir -p %s'" % (strRemoteUserName, strRemoteHostName, strDstFolderName) )
                    strPrevDstFolderName = strDstFolderName
                dstFile = strDstFolderName + dstFile

            if bStoreTmp:
                # store in tmp, send image on the fly
                file = "/tmp/last.jpg" # to store nothing locally enable this line
            else:
                # store locally every images and send last one to remote
                #file = "/home/pi/images/%09.03f.jpg" % time.time()
                strQuality = "%08.03f" % rDiff
                file = "/home/pi/images/" + getFilenameFromTime() + "__" + strQuality + ".jpg"
            cv2.imwrite(file, im,[int(cv2.IMWRITE_JPEG_QUALITY), 50])
            if bStoreTmp:
                sendFileToRemote( "protolab.aldebaran.com", "amazel", file, dstFile )
            else:
                bgs.addFile( file, dstFile )

            if ( time.time() - timeLastAlarmSound ) > 60*10:
                timeLastAlarmSound = time.time()
                launchAlarmSound()

            duration = time.time() - timeBegin
            print( "DBG: time to store: %5.2fs" % duration )

        imPrevGrey = imGrey
        time.sleep( 0.1 ) # leave a bit of cpu to background task

    # You'll want to release the camera, otherwise you won't be able to create a new
    # capture object until your script exits
    del(camera)

# Camera 0 is the integrated web cam on my netbook
camera_port = 0

watchAndSend( camera_port )


from typing import Callable, Dict, List
from redis import StrictRedis
import time
from slackclient import SlackClient
from .data import Bot, Command, Event

class PyBot(SlackClient):

    def __init__(self, token: str, bot: Bot, db: StrictRedis) -> None:
        super().__init__(token)
        if (not self.rtm_connect()):
            raise IOError('Connection to Slack failed, check your token')
        self.bot = bot
        self.db = db
        self._triggers: Dict = {}
        self._USER_CACHE: Dict[str, object] = {}

    def _getCachedUser(self, user_id: int):
        # store a cache of userId -> userName mappings,
        # so we don't need to make API calls every time
        if not self._USER_CACHE.get(user_id):
            self._USER_CACHE[user_id] = self.api_call(
                'users.info', user=user_id
            )['user']
        return self._USER_CACHE[user_id]

    def postMessage(self, channel: str, message: str, thread: str):
        self.api_call(
            'chat.postMessage',
            channel=channel,
            text=message,
            thread_ts=thread,
            username=self.bot.name,
            icon_emoji=self.bot.icon_emoji
        )

    def register(self, trigger: str, callback, condition):
        # registers a trigger, which fires a callback if condition is true
        maybeCallback = _MaybeCallback(callback, condition)
        self._triggers.setdefault(trigger, [maybeCallback])

    def notify(self, command: Command):
        # notifies all subscribers when command triggers, if condition is true
        for mc in (self._triggers.get(command.trigger, [])):
            if mc.condition(command.event):
                mc.callback(command)

    def listen(self):
        # listens for commands, and process them in turn
        while True:
            events = filter(lambda e: e.get('type') ==
                            'message' and 'text' in e, self.rtm_read())
            for event in events:
                command = self._messageEventToCommand(event)
                if command:
                    self.notify(command)  # notifies all listeners
            time.sleep(0.5)

    def _messageEventToCommand(self, event: Event):
        for trigger in self._triggers.keys():
            if event['text'].startswith(trigger):
                args = event['text'][len(trigger):].strip().split()
                return Command(
                    trigger,
                    args,
                    Event(
                        event.get('type'),
                        event.get('subtype'),
                        event.get('channel'),
                        event.get('user'),
                        event.get('text'),
                        event.get('ts'),
                        event.get('thread_ts')
                    ),
                    self._getCachedUser(event.user_id)
                )

        return None

def threaded(event):
    return allMessageEvents(event) and event.thread is not None

def messageEvents(event):
    return allMessageEvents(event) and event.thread is None

def allMessageEvents(event):
    return (
        event.type == 'message' and
        event.subtype is None and
        event.text is not None
    )

class _MaybeCallback(object):
    def __init__(self, callback, condition):
        self.callback = callback
        self.condition = condition


from ..helper import config, get_class_from_dot_string
from ..logging import logger

default_storage = config('DEFAULT_FILE_STORAGE')

class DefautStorageBackend:
    default_backend = None
    instance = None

    def __init__(self):
        if not default_storage:
            raise Exception('Default storage backend not configured')

        self.default_backend = get_class_from_dot_string(default_storage)()

    @property
    def backend(self):
        return self.default_backend

    @staticmethod
    def getInstance():
        if not DefautStorageBackend.instance:
            DefautStorageBackend.instance = (DefautStorageBackend()).backend
        return DefautStorageBackend.instance

try:
    default_backend = DefautStorageBackend.getInstance()
except Exception as e:
    logger.exception(e)
    default_backend = None

__all__ = [
    'default_backend'
]