import pandas as pd
import datetime
import numpy
import vfb_connect
from vfb_connect.cross_server_tools import VfbConnect
import pysolr
import json
import itertools
passed = {'hair plate': 'mechanosensory neuron of hair plate', 'campaniform sensillum': 'sensory neuron of campaniform sensillum', 'T3 leg club chordotonal neuron': 'metathoracic femoral chordotonal club neuron', 'T2 leg claw chordotonal neuron': 'mesothoracic femoral chordotonal claw neuron', 'right T1 ventral nerve': 'adult ventral prothoracic nerve',
          'left T1 ventral nerve': 'adult ventral prothoracic nerve', 'T1 leg claw chordotonal neuron': 'prothoracic femoral chordotonal claw neuron', 'T1 leg club chordotonal neuron': 'prothoracic femoral chordotonal club neuron', 'T1 leg hook chordotonal neuron': 'prothoracic femoral chordotonal hook neuron',
          'haltere motor neuron HN bundle':'adult dorsal metathoracic nerve','left T1 dorsal nerve':'adult dorsal prothoracic nerve','right T1 dorsal nerve':'adult dorsal prothoracic nerve','bCS':'bilateral campaniform sensillum neuron of leg','CoHP8':'mechanosensory neuron of prothoracic coxal hair plate CoHP8',
          'T2 leg motor neuron PDMN bundle':'adult posterior dorsal mesothoracic nerve'}
missing = {}
used = []
ref_terms = ['UPDATED', 'LINKED', 'Paper', 'et al.', ' from ',
             '?', 'on server', 'assigned to', 'update ', 'need to ']

def find_offical_label(term):
    solr = pysolr.Solr('https://solr.virtualflybrain.org/solr/ontology/')
    for ref in ref_terms:
        if ref in term:
            return ''
    if term in missing.keys():
        return ''
    if term in passed.keys():
        return passed[term]
    if "FBbt" in term:
        results = solr.search('shortform_autosuggest:"' + term + '"')
        for doc in results.docs:
            try:
                if term in doc['shortform_autosuggest'] and 'FBbt' in doc['short_form']:
                    passed.update({term: doc['label']})
                    return doc['label']
            except:
                print('FBbt not matched:' + json.dumps(doc, indent = 4))
    expanded = []
    expanded.append(term)
    replacements = {'T1 ': 'prothoracic ',
                    'T3 ': 'metathoracic ', 'T2 ': 'mesothoracic ', 'wing ':'flight '}
    modified = term
    for rep in replacements:
        modified = modified.replace(rep, replacements[rep])
    expanded.append(modified)
    if not ' soma' in term and ('left ' in term or 'right ' in term):
        expanded.append(term.replace('left ', '').replace('right ', ''))
        expanded.append('adult ' + term.replace('left ',
                        '').replace('right ', ''))
        expanded.append(modified.replace('left ', '').replace('right ', ''))
        expanded.append('adult ' + modified.replace('left ',
                        '').replace('right ', ''))
    expanded.append('adult ' + term)  # TODO check stage
    expanded.append('adult ' + modified)
    if 'DUM' in term:
        expanded.append(term.replace('DUM', 'dorsal unpaired median'))
        expanded.append(modified.replace('DUM', 'dorsal unpaired median'))
    if not ' ' in term:
        expanded.append(term + ' neuron')
        expanded.append(modified + ' neuron')
        expanded.append('adult ' + term + ' neuron')
        expanded.append('adult ' + modified + ' neuron')
    if ' motor neuron' in term:
        expanded.append(term.replace(' motor neuron',' muscle motor neuron'))
        expanded.append(modified.replace(' motor neuron',' muscle motor neuron'))
    expanded = list(set(expanded))
    expanded.sort(key=len, reverse=True)
    for test in expanded:
        results = solr.search('label:"' + test + '" OR synonym:"' + test + '"')
        for doc in results.docs:
          try:
                if test in doc['synonym'] and 'FBbt' in doc['short_form']:
                    passed.update({term: doc['label']})
                    return doc['label']
          except:
                print('synonym missing from:' + json.dumps(doc, indent = 4))
    if not term in missing.keys() and not ':' in term:
        missing[term] = expanded
    return ''

def resolve_entity(entity="", annotation_series=[]):
    if not 'adult ventral nerve cord' in entity:
        return entity
    entityRows = []
    if 'adult ventral nerve cord' in entity:
        for annotations in annotation_series:
            if 'motor neuron' in annotations or 'sensory neuron' in annotations or 'nerve' in annotations or 'bundle' in annotations:
                entityRows.append(entity.replace(
                    'ventral nerve cord', 'nervous system'))
            else:
                entityRows.append(entity)
        return pd.Series(entityRows)
    return entity

def find_available_terms(annotation_series=[]):
    """
    Return a '|' delimited list of annotation labels that exists as an FBbt term in pdb
    """
    vc = VfbConnect(neo_endpoint='http://pdb.virtualflybrain.org')
    results = []
    for annotations in annotation_series:
        names = []
        result = "neuron"
        for annotation in annotations.split('), '):
            if not annotation.split(' (')[0] in names and not annotation.split(' (')[0] == "neuron" and not 'nerve' in annotation.split(' (')[0] and not 'bundle' in annotation.split(' (')[0]:
                name = find_offical_label(annotation.split(' (')[0])
                if len(name) > 0 and not name in names:
                    names.append(name)
        for name in names:
            try:
                id = vc.lookup_id(name)
                if 'FBbt' in id:
                    result += '|' + name
                else:
                    if not name in missing.keys():
                        missing[name] = annotations
            except:
                if not name in missing.keys():
                    missing[name] = annotations
        results.append(result)
    return pd.Series(results)

def find_dbxrefs(annotation_series=[]):
    """
    Return a '|' delimited Series of dbxrefs form annotaions
    """
    xrefs = []
    for annotations in annotation_series:
        result = ""
        for annotation in annotations.split('), '):
            name = annotation.split(' (')[0]
            if 'project id 2 on server https://catmaid3.hms.harvard.edu/catmaidvnc' in name:
                if 'LINKED NEURON - elastic transformation and flipped of skeleton id ' in name:
                    result += "|" + name.replace('LINKED NEURON - elastic transformation and flipped of skeleton id ', 'catmaid_fanc:').replace(
                        ' in project id 2 on server https://catmaid3.hms.harvard.edu/catmaidvnc', '')
                else:
                    result += "|" + name.replace('LINKED NEURON - elastic transformation of skeleton id ', 'catmaid_fanc:').replace(
                        ' in project id 2 on server https://catmaid3.hms.harvard.edu/catmaidvnc', '')
            if 'source: https://gen1mcfo.janelia.org/cgi-bin/view_gen1mcfo_imagery.cgi?line=' in name:
                result += "|" + \
                    name.replace(
                        'source: https://gen1mcfo.janelia.org/cgi-bin/view_gen1mcfo_imagery.cgi?line=', 'FlyLightGen1MCFO:')
            if 'project id 59 on server https://catmaid3.hms.harvard.edu/catmaidvnc' in name:
                if 'LINKED NEURON - pruned (first entry, last exit) by vol 109 of skeleton id ' in name:
                    result += "|" + name.replace('LINKED NEURON - pruned (first entry, last exit) by vol 109 of skeleton id ',
                                                 'catmaid_fanc_JRC2018VF:').replace(' in project id 59 on server https://catmaid3.hms.harvard.edu/catmaidvnc', '')
                if 'LINKED NEURON - radius pruned of skeleton id ' in name:
                    result += "|" + name.replace('LINKED NEURON - radius pruned of skeleton id ', 'catmaid_fanc_JRC2018VF:').replace(
                        ' in project id 59 on server https://catmaid3.hms.harvard.edu/catmaidvnc', '')
        xrefs.append(result)
    return pd.Series(xrefs)

def generate_comments(annotation_series=[]):
    """
    Return a ', ' delimited Series of comments form annotaions
    """
    comments = []
    for annotations in annotation_series:
        result = ""
        for annotation in annotations.split('), '):
            name = annotation.split(' (')[0]
            for ref in ref_terms:
                if ref in name:
                    name = ''
            if len(result) > 0 and len(name) > 0:
                name = ', ' + name
            if ': ' in name and not 'Paper:' in name:
                result += name
            else:
                for use in [' from ', ' soma', 'pruned', ' flipped', 'nerve', 'bundle']:
                    if use in name and not name in result:
                        result += name
                        break
                if not name.replace(', ', '') in result and not name in result:
                    result += name
        comments.append(result)
    return pd.Series(comments)

def create_metadata(db="", filename_series=[], annotation_series=[], pub=""):
    results = []
    soma_rules = {'left soma': 'left side of organism', 'right soma': 'right side of organism',
                  'midline soma': 'cell body rind along midline of adult ventral nerve cord'}
    nerve_rules = {}
    for id, annotations in zip(filename_series, annotation_series):
        if ' soma' in annotations:
            for key in soma_rules.keys():
                if key in annotations:
                    results.append({'object': soma_rules[key], 'relation': 'has soma location',
                                   'subject_external_id': id, 'subject_external_db': db, 'pub': pub})
        else:
            if 'left ' in annotations:
                results.append({'object': 'left side of organism', 'relation': 'overlaps',
                               'subject_external_id': id, 'subject_external_db': db, 'pub': pub})
            if 'right ' in annotations:
                results.append({'object': 'right side of organism', 'relation': 'overlaps',
                               'subject_external_id': id, 'subject_external_db': db, 'pub': pub})
        if 'nerve' in annotations or 'bundle' in annotations:
            for annotation in annotations.split('), '):
                name = find_offical_label(annotation.split(' (')[0])
                if len(name) > 0 and ('nerve' in name or 'nerve' in annotation):
                    results.append({'object': name, 'relation': 'fasciculates_with',
                                   'subject_external_id': id, 'subject_external_db': db, 'pub': pub})
        if 'motor neuron' in annotations or 'sensory neuron' in annotations or 'nerve' in annotations:
            results.append({'object': 'peripheral nervous system', 'relation': 'overlaps',
                           'subject_external_id': id, 'subject_external_db': db, 'pub': pub})
            results.append({'object': 'adult ventral nerve cord', 'relation': 'overlaps',
                           'subject_external_id': id, 'subject_external_db': db, 'pub': pub})
        if 'bilateral' in annotations:
            results.append({'object': 'bilateral', 'relation': 'has_characteristic',
                           'subject_external_id': id, 'subject_external_db': db, 'pub': pub})
        if 'contains presynapses' in annotations:
            results.append({'object': 'adult ventral nerve cord', 'relation': 'has_presynaptic_terminal_in',
                           'subject_external_id': id, 'subject_external_db': db, 'pub': pub})
    return pd.DataFrame(results)

def make_anat_records(site, curator, output_filename='./anat'):
    """
    Makes image curation record(s) for new skids in VFB_reporting_results/<site>_new_skids.tsv.
    These should be transferred to curation repo for loading.

    Ignores any skids whose catmaid paper ID is not associated with a DataSet in VFB.
    This should not be automatically run every day (makes dated files for curation).
    """
    if site not in ['L1EM', 'FAFB', 'FANC1', 'FANC2', 'VNC1', 'LEG40']:
        raise KeyError('site must be L1EM, FAFB, FANC1, FANC2 OR LEG40')
    # date for filenames
    today = datetime.date.today()
    datestring = today.strftime("%y%m%d")

    save_directory = "../VFB_reporting_results/CATMAID_SKID_reports/"

    # open file of new skids
    new_skids = pd.read_csv("%s%s_new_skids.tsv" % (save_directory, site), sep='\t')\
        .applymap(str)
    paper_ids = set(list(new_skids['paper_id']))

    # get mapping of ds name (in VFB) to id (as index) from <site>_comparison.tsv
    comparison_table = pd.read_csv("%s%s_comparison.tsv" % (save_directory, site),
                                   sep='\t', index_col='Paper_ID').applymap(str)
    comparison_table.index = comparison_table.index.map(str)

    # get dataset name for paper from vfb
    for i in paper_ids:
        ds = comparison_table['VFB_name'][i]  # dataset name in VFB
        if ds == str(numpy.nan):
            continue  # if no VFB dataset for paper

        single_ds_data = new_skids[new_skids['paper_id'] == i]
        single_ds_data = single_ds_data.reset_index()
        image_type = 'TEM'
        if site == 'L1EM':
            entity = 'embryonic/larval nervous system'
            template = 'L1 larval CNS ssTEM - Cardona/Janelia_c'
            instance = 'L1EM'
        elif site == 'FAFB':
            entity = 'female organism|adult brain'
            template = 'JRC2018Unisex_c'
            instance = 'FAFB'
        elif site in ['FANC1', 'FANC2']:
            entity = 'female organism|adult ventral nerve cord'
            template = 'JRC2018UnisexVNC_c'
            instance = 'FANC'
        elif site == 'LEG40':
            entity = 'female organism|prothoracic leg'
            template = 'T1Leg2020_c'
            instance = 'T1LEG'
            image_type = 'XNH'
        if site == 'FAFB':
            curation_df = pd.DataFrame({'filename': single_ds_data['skid'],
                                    'label': single_ds_data['name'].map(
                                        lambda x: str('%s' % (x.replace(' - elastic transform', '')))),
                                    'is_a': 'neuron',
                                    'part_of': resolve_entity(entity, single_ds_data['annotations']),
                                    'comment': generate_comments(single_ds_data['annotations'])})
        elif site == 'LEG40':
            curation_df = pd.DataFrame({'filename': single_ds_data['skid'],
                                    'label': single_ds_data['name'].map(
                                        lambda x: str('%s' % (x.replace(' - elastic transform', '')))),
                                    'is_a': find_available_terms(single_ds_data['name'].map(
                                        lambda x: str('%s' % ("".join(itertools.dropwhile(str.isdigit, x)).strip())))),
                                    'part_of': resolve_entity(entity, single_ds_data['annotations']),
                                    'comment': generate_comments(single_ds_data['annotations'])})
        else:
            curation_df = pd.DataFrame({'filename': single_ds_data['skid'],
                                    'label': single_ds_data['name'].map(
                                        lambda x: str('%s' % (x.replace(' - elastic transform', '')))),
                                    'is_a': find_available_terms(single_ds_data['annotations']),
                                    'part_of': resolve_entity(entity, single_ds_data['annotations']),
                                    'comment': generate_comments(single_ds_data['annotations'])})
        curation_df['dbxrefs'] = curation_df['filename'].map(
            lambda x: str('catmaid_%s:%s' % (site.lower().replace('fanc2', 'fanc_JRC2018VF'), str(x))))
        curation_df['dbxrefs'] += find_dbxrefs(single_ds_data['annotations'])
        if 'synonyms' in single_ds_data.keys():
            curation_df['synonyms'] = single_ds_data['synonyms'].fillna('', inplace=True)
        if single_ds_data['skid'].to_string() not in curation_df['label'].to_string():
            curation_df['label'] = curation_df['label'] + \
                ' (' + instance + ':' + single_ds_data['skid'] + ')'
#         curation_df['label'] = curation_df[['label', 'filename']].apply(lambda x: ' '.join(x), axis=1)

        if output_filename == "./anat":
            output_filename1 = './anat_%s_%s' % (ds, datestring)
        else:
            output_filename1 = output_filename + '_' + ds
        curation_df.to_csv(output_filename1 + '.tsv', sep='\t', index=None)
        output_filename2 = output_filename + '_' + ds
        create_metadata(db="catmaid_" + site.lower().replace('fanc2', 'fanc_JRC2018VF').replace('fanc1','fanc'), filename_series=single_ds_data['skid'], annotation_series=single_ds_data['annotations']).to_csv(
            output_filename1.replace('anat_', 'meta_') + '.tsv', sep='\t', index=None)

        with open(output_filename1 + '.yaml', 'w') as file:
            file.write("DataSet: %s\n" % ds)
            file.write("Curator: %s\n" % curator)
            file.write("Imaging_type: %s\n" % image_type)
            file.write("Template: %s\n" % template)

if __name__ == "__main__":
    make_anat_records('FAFB', 'travis',
                      '../VFB_reporting_results/anat_fafb_missing')
    missing.clear()  # not used YET on ^
    make_anat_records('L1EM', 'travis',
                      '../VFB_reporting_results/anat_l1em_missing')
    missing.clear()  # not used YET on ^
    make_anat_records('FANC1', 'travis',
                      '../VFB_reporting_results/anat_fanc1_missing')
    pd.DataFrame({'missing terms': pd.Series(missing.keys()), 'meta': pd.Series(missing.values())}).to_csv(
        '../VFB_reporting_results/CATMAID_SKID_reports/anat_fanc1_missing_terms.tsv', sep='\t', index=None)
    pd.DataFrame({'CATMAID annotation': pd.Series(passed.keys()), 'FBbt term': pd.Series(passed.values())}).to_csv(
        '../VFB_reporting_results/CATMAID_SKID_reports/anat_fanc1_resolved_terms.tsv', sep='\t', index=None)
    missing.clear()
    make_anat_records('FANC2', 'travis',
                      '../VFB_reporting_results/anat_fanc2_missing')
    pd.DataFrame({'missing terms': pd.Series(missing.keys()), 'meta': pd.Series(missing.values())}).to_csv(
        '../VFB_reporting_results/CATMAID_SKID_reports/anat_fanc2_missing_terms.tsv', sep='\t', index=None)
    pd.DataFrame({'CATMAID annotation': pd.Series(passed.keys()), 'FBbt term': pd.Series(passed.values())}).to_csv(
        '../VFB_reporting_results/CATMAID_SKID_reports/anat_fanc2_resolved_terms.tsv', sep='\t', index=None)
    make_anat_records('LEG40', 'travis',
                      '../VFB_reporting_results/anat_leg40_missing')
    pd.DataFrame({'missing terms': pd.Series(missing.keys()), 'meta': pd.Series(missing.values())}).to_csv(
        '../VFB_reporting_results/CATMAID_SKID_reports/anat_leg40_missing_terms.tsv', sep='\t', index=None)
    pd.DataFrame({'CATMAID annotation': pd.Series(passed.keys()), 'FBbt term': pd.Series(passed.values())}).to_csv(
        '../VFB_reporting_results/CATMAID_SKID_reports/anat_leg40_resolved_terms.tsv', sep='\t', index=None)


import os
import requests
import sys
import json
from bs4 import BeautifulSoup

BASE_DIR = os.path.dirname('__File__')
print(BASE_DIR)

print("Start Crawling")
req = requests.get('https://music.naver.com/listen/top100.nhn?domain=TOTAL')
req.encoding = None
html = req.content #html요소 전부 가져오기
soup = BeautifulSoup(html, 'html.parser') #html 파싱
datas = soup.select(
    '#content > div.NE\=a\:t1a > div._tracklist_mytrack.tracklist_table.tracklist_type1 > table > tbody > tr > td.name > a._title > span'
    )
data = {}

for i in range(len(datas)):
    rank = i+1
    name = datas[i].get_text()
    print(str(rank) + ". " + name)
    data[rank] = name

with open(os.path.join(BASE_DIR, 'musicRank.json'), 'w+',encoding='utf-8') as json_file:
    json.dump(data, json_file, ensure_ascii = False, indent='\t')

print("Crawling Complete")


import unittest

import os
import shutil

import attr

from handwriting import run_charclassml, run_charposml
from handwriting import config as cf

class TestML(unittest.TestCase):
    """Integration tests for machine learning training processes"""

    def test_run_charclassml(self):
        """test run_charclassml"""
        config = attr.assoc(run_charclassml.CONFIG_DEFAULT)
        config.nn_opt["max_epochs"] = 1
        config.train.idxs = [1]
        config.train.do_balance = True
        config.train.balance_size = 64
        config.dev.idxs = [2]
        config.test.idxs = [3]
        self._test_training_module(
            run_charclassml,
            "charclass",
            config)

    def test_run_charposml(self):
        """test run_charposml"""
        run_charposml.VERBOSE = True
        config = attr.assoc(run_charposml.CONFIG_DEFAULT)
        config.train.idxs = [1]
        config.nn_opt["max_epochs"] = 1
        config.train.do_balance = True
        config.train.balance_size = 64
        config.dev.idxs = [2]
        config.test.idxs = [3]
        self._test_training_module(
            run_charposml,
            "charpos",
            config)

    def _test_training_module(self, module, sub_dirname, config):
        """helper method: run one of the training modules with config,
        testing that it produces output files"""

        # this function might not make sense later if the training modules
        # get different configurations or have different types of outputs

        work_dirname = os.path.join("integration", sub_dirname)

        if os.path.exists(work_dirname):
            shutil.rmtree(work_dirname)
        os.makedirs(work_dirname)

        # generate config file
        config_filename = os.path.join(work_dirname, "config.json")
        cf.save(config, config_filename)
        model_filename = os.path.join(work_dirname, "model.pkl")
        args = ["", "train", config_filename, model_filename]

        module.main(args)

        # assert that output model and log exist
        self.assertTrue(os.path.exists(model_filename))
        self.assertTrue(os.path.exists(model_filename + ".log.txt"))