# -*- coding: utf-8 -*-
"""
Note:
    THERE ARE FUNCTIONS THAT ARE INJECTED INTO THE CONTROLLER
    THAT ARE NOT DEFINED IN THIS MODULE.

    functions in the IBEISController have been split up into several submodules.
    look at the modules listed in autogenmodname_list to see the full list of
    functions that will be injected into an IBEISController object

TODO:
    Module Licence and docstring
"""
# TODO: rename annotation annotations
# TODO: make all names consistent
from __future__ import absolute_import, division, print_function
# Python
import six
import atexit
import requests
import weakref
from six.moves import zip
from os.path import join, split
import utool as ut
import ibeis  # NOQA
from ibeis.init import sysres
from ibeis import constants as const
#from ibeis import params
from ibeis.control import accessor_decors, controller_inject
from ibeis.control.accessor_decors import (default_decorator, )
import xml.etree.ElementTree as ET
# Import modules which define injectable functions
# Older manual ibeiscontrol functions
from ibeis import ibsfuncs
from ibeis.model.hots import pipeline

# NOTE: new plugin code needs to be hacked in here currently
# this is not a long term solution.

# <Pyinstaller hacks>
from ibeis.control import _autogen_featweight_funcs  # NOQA
from ibeis.control import _autogen_party_funcs  # NOQA
from ibeis.control import _autogen_annotmatch_funcs  # NOQA
from ibeis.control import manual_ibeiscontrol_funcs  # NOQA
from ibeis.control import manual_wildbook_funcs  # NOQA
from ibeis.control import manual_meta_funcs  # NOQA
from ibeis.control import manual_lbltype_funcs  # NOQA   # DEPRICATE
from ibeis.control import manual_lblannot_funcs  # NOQA  # DEPRICATE
from ibeis.control import manual_lblimage_funcs  # NOQA  # DEPRICATE
from ibeis.control import manual_image_funcs  # NOQA
from ibeis.control import manual_encounter_funcs  # NOQA
from ibeis.control import manual_egrelate_funcs  # NOQA
from ibeis.control import manual_garelate_funcs  # NOQA
from ibeis.control import manual_annot_funcs  # NOQA
from ibeis.control import manual_name_funcs  # NOQA
from ibeis.control import manual_species_funcs  # NOQA
from ibeis.control import manual_annotgroup_funcs  # NOQA
#from ibeis.control import manual_dependant_funcs  # NOQA
from ibeis.control import manual_chip_funcs  # NOQA
from ibeis.control import manual_feat_funcs  # NOQA
# </Pyinstaller hacks>

# Shiny new way to inject external functions
autogenmodname_list = [
    ('ibeis.control', '_autogen_featweight_funcs'),
    ('ibeis.control', '_autogen_party_funcs'),
    ('ibeis.control', '_autogen_annotmatch_funcs'),
    # ('ibeis.control', '_autogen_annot_funcs'),
    ('ibeis.control', 'manual_ibeiscontrol_funcs'),
    ('ibeis.control', 'manual_wildbook_funcs'),
    ('ibeis.control', 'manual_meta_funcs'),
    ('ibeis.control', 'manual_lbltype_funcs'),   # DEPRICATE
    ('ibeis.control', 'manual_lblannot_funcs'),  # DEPRICATE
    ('ibeis.control', 'manual_lblimage_funcs'),  # DEPRICATE
    ('ibeis.control', 'manual_image_funcs'),
    ('ibeis.control', 'manual_encounter_funcs'),
    ('ibeis.control', 'manual_egrelate_funcs'),
    ('ibeis.control', 'manual_garelate_funcs'),
    ('ibeis.control', 'manual_annot_funcs'),
    ('ibeis.control', 'manual_name_funcs'),
    ('ibeis.control', 'manual_species_funcs'),
    ('ibeis.control', 'manual_annotgroup_funcs'),
    # ('ibeis.control', 'manual_dependant_funcs'),
    ('ibeis.control', 'manual_chip_funcs'),
    ('ibeis.control', 'manual_feat_funcs'),
]

WITH_CNN = ut.get_argflag(('--with-cnn', '--withcnn', '--cnn'))

# HACK, don't include cnn unless its already there due to theano stuff
pluginmodname_list = []
import sys
if 'ibeis_cnn' in sys.modules or WITH_CNN:
    pluginmodname_list = [
        ('ibeis_cnn', '_plugin'),
    ]

def make_explicit_imports_for_pyinstaller():
    #making actual imports pyinstaller
    print('\n'.join(['from %s import %s  # NOQA' % pkgtup for pkgtup in autogenmodname_list]))

INJECTED_MODULES = []

for pkgname, modname in autogenmodname_list + pluginmodname_list:
    try:
        exec('from %s import %s' % (pkgname, modname, ), globals(), locals())
        module = eval(modname)
        INJECTED_MODULES.append(module)
    except ImportError as ex:
        ut.printex(ex, 'Cannot load package=%r, module=%r' % (pkgname, modname, ))

# Inject utool functions
(print, print_, printDBG, rrr, profile) = ut.inject(__name__, '[ibs]')

register_api   = controller_inject.get_ibeis_flask_api(__name__)
register_route = controller_inject.get_ibeis_flask_route(__name__)

__ALL_CONTROLLERS__ = []  # Global variable containing all created controllers
__IBEIS_CONTROLLER_CACHE__ = {}

def request_IBEISController(
        dbdir=None, ensure=True, wbaddr=None, verbose=ut.VERBOSE,
        use_cache=True, request_dbversion=None):
    r"""
    Alternative to directory instantiating a new controller object. Might
    return a memory cached object

    Args:
        dbdir     (str): databse directory
        ensure    (bool):
        wbaddr    (None):
        verbose   (bool):
        use_cache (bool): use the global ibeis controller cache. (default=True)
        request_dbversion (str): developer flag. Do not use.

    Returns:
        IBEISController: ibs

    CommandLine:
        python -m ibeis.control.IBEISControl --test-request_IBEISController

    Example:
        >>> # ENABLE_DOCTEST
        >>> from ibeis.control.IBEISControl import *  # NOQA
        >>> dbdir = 'testdb1'
        >>> ensure = True
        >>> wbaddr = None
        >>> verbose = True
        >>> use_cache = False
        >>> ibs = request_IBEISController(dbdir, ensure, wbaddr, verbose, use_cache)
        >>> result = str(ibs)
        >>> print(result)
    """
    # TODO: change name from new to request
    global __IBEIS_CONTROLLER_CACHE__
    if use_cache and dbdir in __IBEIS_CONTROLLER_CACHE__:
        if verbose:
            print('[request_IBEISController] returning cached controller')
        ibs = __IBEIS_CONTROLLER_CACHE__[dbdir]
    else:
        # Convert hold hotspotter dirs if necessary
        from ibeis.dbio import ingest_hsdb
        if ingest_hsdb.check_unconverted_hsdb(dbdir):
            ibs = ingest_hsdb.convert_hsdb_to_ibeis(dbdir, ensure=ensure, wbaddr=wbaddr, verbose=verbose)
        else:
            ibs = IBEISController(
                dbdir=dbdir, ensure=ensure, wbaddr=wbaddr, verbose=verbose,
                request_dbversion=request_dbversion)
        __IBEIS_CONTROLLER_CACHE__[dbdir] = ibs
    return ibs

@atexit.register
def __cleanup():
    """ prevents flann errors (not for cleaning up individual objects) """
    global __ALL_CONTROLLERS__
    global __IBEIS_CONTROLLER_CACHE__
    try:
        del __ALL_CONTROLLERS__
        del __IBEIS_CONTROLLER_CACHE__
    except NameError:
        print('cannot cleanup IBEISController')
        pass

#
#
#-----------------
# IBEIS CONTROLLER
#-----------------

@six.add_metaclass(ut.ReloadingMetaclass)
class IBEISController(object):
    """
    IBEISController docstring

    NameingConventions:
        chip  - cropped region of interest in an image, maps to one animal
        cid   - chip unique id
        gid   - image unique id (could just be the relative file path)
        name  - name unique id
        eid   - encounter unique id
        aid   - region of interest unique id
        annot - an annotation i.e. region of interest for a chip
        theta - angle of rotation for a chip
    """

    #
    #
    #-------------------------------
    # --- CONSTRUCTOR / PRIVATES ---
    #-------------------------------

    def __init__(ibs, dbdir=None, ensure=True, wbaddr=None, verbose=True,
                 request_dbversion=None):
        """ Creates a new IBEIS Controller associated with one database """
        #if verbose and ut.VERBOSE:
        print('\n[ibs.__init__] new IBEISController')
        # an dict to hack in temporary state
        ibs.const = const
        ibs.allow_override = 'override+warn'
        # observer_weakref_list keeps track of the guibacks connected to this controller
        ibs.observer_weakref_list = []
        # not completely working decorator cache
        ibs.table_cache = None
        ibs._initialize_self()
        ibs._init_dirs(dbdir=dbdir, ensure=ensure)
        # _send_wildbook_request will do nothing if no wildbook address is specified
        ibs._send_wildbook_request(wbaddr)
        ibs._init_sql(request_dbversion=request_dbversion)
        ibs._init_config()
        print('[ibs.__init__] END new IBEISController\n')

    def reset_table_cache(ibs):
        ibs.table_cache = accessor_decors.init_tablecache()

    def clear_table_cache(ibs, tablename=None):
        print('[ibs] clearing table_cache[%r]' % (tablename,))
        if tablename is None:
            ibs.reset_table_cache()
        else:
            del ibs.table_cache[tablename]

    def get_cachestats_str(ibs):
        """
        Returns info about the underlying SQL cache memory
        """
        total_size_str = ut.get_object_size_str(ibs.table_cache, lbl='size(table_cache): ')
        total_size_str = '\nlen(table_cache) = %r' % (len(ibs.table_cache))
        table_size_str_list = [ut.get_object_size_str(val, lbl='size(table_cache[%s]): ' % (key,))
                               for key, val in six.iteritems(ibs.table_cache)]
        cachestats_str = total_size_str + ut.indentjoin(table_size_str_list, '\n  * ')
        return cachestats_str

    def print_cachestats_str(ibs):
        cachestats_str = ibs.get_cachestats_str()
        print('IBEIS Controller Cache Stats:')
        print(cachestats_str)
        return cachestats_str

    #def inject_module_plugin(ibs, module):
    #    global INJECTED_MODULES
    #    if module not in INJECTED_MODULES:
    #        INJECTED_MODULES.append(module)
    #    ut.inject_instance(
    #        ibs, classtype=module.CLASS_INJECT_KEY,
    #        allow_override=ibs.allow_override, strict=False)

    def _initialize_self(ibs):
        """
        For utools auto reload
        Called after reload
        Injects code from development modules into the controller
        """
        if ut.VERBOSE:
            print('[ibs] _initialize_self()')
        ibs.reset_table_cache()

        for module in INJECTED_MODULES:
            ut.inject_instance(
                ibs, classtype=module.CLASS_INJECT_KEY,
                allow_override=ibs.allow_override, strict=False)
        ut.inject_instance(ibs, classtype=ibsfuncs.CLASS_INJECT_KEY,
                           allow_override=ibs.allow_override, strict=True)
        assert hasattr(ibs, 'get_database_species'), 'issue with ibsfuncs'

        #ut.inject_instance(ibs, classtype=('IBEISController', 'autogen_featweight'),
        #                   allow_override=ibs.allow_override, strict=False)
        #ut.inject_instance(ibs, classtype=('IBEISController', 'manual'),
        #                   allow_override=ibs.allow_override, strict=False)
        ibs.register_controller()

    def _on_reload(ibs):
        """
        For utools auto reload.
        Called before reload
        """
        # Reloading breaks flask, turn it off
        controller_inject.GLOBAL_APP_ENABLED = False
        # Only warn on first load. Overrideing while reloading is ok
        ibs.allow_override = True
        ibs.unregister_controller()
        # Reload dependent modules
        for module in INJECTED_MODULES:
            module.rrr()
        ibsfuncs.rrr()
        pass

    # We should probably not implement __del__
    # see: https://docs.python.org/2/reference/datamodel.html#object.__del__
    #def __del__(ibs):
    #    ibs.cleanup()

    # ------------
    # SELF REGISTRATION
    # ------------

    def register_controller(ibs):
        """ registers controller with global list """
        ibs_weakref = weakref.ref(ibs)
        __ALL_CONTROLLERS__.append(ibs_weakref)

    def unregister_controller(ibs):
        ibs_weakref = weakref.ref(ibs)
        try:
            __ALL_CONTROLLERS__.remove(ibs_weakref)
            pass
        except ValueError:
            pass

    # ------------
    # OBSERVER REGISTRATION
    # ------------

    def cleanup(ibs):
        """ call on del? """
        print('[ibs.cleanup] Observers (if any) notified [controller killed]')
        for observer_weakref in ibs.observer_weakref_list:
            observer_weakref().notify_controller_killed()

    @default_decorator
    def register_observer(ibs, observer):
        print('[register_observer] Observer registered: %r' % observer)
        observer_weakref = weakref.ref(observer)
        ibs.observer_weakref_list.append(observer_weakref)

    @default_decorator
    def remove_observer(ibs, observer):
        print('[remove_observer] Observer removed: %r' % observer)
        ibs.observer_weakref_list.remove(observer)

    @default_decorator
    def notify_observers(ibs):
        print('[notify_observers] Observers (if any) notified')
        for observer_weakref in ibs.observer_weakref_list:
            observer_weakref().notify()

    # ------------

    def _init_rowid_constants(ibs):
        ibs.UNKNOWN_LBLANNOT_ROWID = 0  # ADD TO CONSTANTS
        ibs.UNKNOWN_NAME_ROWID     = ibs.UNKNOWN_LBLANNOT_ROWID  # ADD TO CONSTANTS
        ibs.UNKNOWN_SPECIES_ROWID  = ibs.UNKNOWN_LBLANNOT_ROWID  # ADD TO CONSTANTS
        ibs.MANUAL_CONFIG_SUFFIX = 'MANUAL_CONFIG'
        ibs.MANUAL_CONFIGID = ibs.add_config(ibs.MANUAL_CONFIG_SUFFIX)
        # duct_tape.fix_compname_configs(ibs)
        # duct_tape.remove_database_slag(ibs)
        # duct_tape.fix_nulled_yaws(ibs)
        lbltype_names    = const.KEY_DEFAULTS.keys()
        lbltype_defaults = const.KEY_DEFAULTS.values()
        lbltype_ids = ibs.add_lbltype(lbltype_names, lbltype_defaults)
        ibs.lbltype_ids = dict(zip(lbltype_names, lbltype_ids))

    @default_decorator
    def _init_sql(ibs, request_dbversion=None):
        """ Load or create sql database """
        from ibeis.other import duct_tape  # NOQA
        ibs._init_sqldbcore(request_dbversion=request_dbversion)
        ibs._init_sqldbcache()
        # ibs.db.dump_schema()
        # ibs.db.dump()
        ibs._init_rowid_constants()

    #@ut.indent_func
    def _init_sqldbcore(ibs, request_dbversion=None):
        """
        Example:
            >>> # DISABLE_DOCTEST
            >>> from ibeis.control.IBEISControl import *  # NOQA
            >>> import ibeis  # NOQA
            >>> #ibs = ibeis.opendb('PZ_MTEST')
            >>> #ibs = ibeis.opendb('PZ_Master0')
            >>> ibs = ibeis.opendb('testdb1')
            >>> #ibs = ibeis.opendb('PZ_Master0')

        Ignore:
            aid_list = ibs.get_valid_aids()
            #ibs.update_annot_visual_uuids(aid_list)
            vuuid_list = ibs.get_annot_visual_uuids(aid_list)
            aid_list2 =  ibs.get_annot_aids_from_visual_uuid(vuuid_list)
            assert aid_list2 == aid_list
            # v1.3.0 testdb1:264us, PZ_MTEST:3.93ms, PZ_Master0:11.6s
            %timeit ibs.get_annot_aids_from_visual_uuid(vuuid_list)
            # v1.3.1 testdb1:236us, PZ_MTEST:1.83ms, PZ_Master0:140ms

            ibs.print_encounter_table(exclude_columns=['encounter_uuid'])

        """
        from ibeis.control import _sql_helpers
        from ibeis.control import SQLDatabaseControl as sqldbc
        from ibeis.control import DB_SCHEMA
        # Before load, ensure database has been backed up for the day
        if not ut.get_argflag('--nobackup'):
            try:
                _sql_helpers.ensure_daily_database_backup(ibs.get_ibsdir(),
                                                          ibs.sqldb_fname,
                                                          ibs.backupdir)
            except IOError as ex:
                ut.printex(ex, (
                    'Failed making daily backup. '
                    'Run with --nobackup to disable'))
                raise
        # IBEIS SQL State Database
        #ibs.db_version_expected = '1.1.1'
        if request_dbversion is None:
            ibs.db_version_expected = '1.4.5'
        else:
            ibs.db_version_expected = request_dbversion
        # TODO: add this functionality to SQLController
        new_version, new_fname = sqldbc.dev_test_new_schema_version(
            ibs.get_dbname(), ibs.get_ibsdir(),
            ibs.sqldb_fname, ibs.db_version_expected, version_next='1.4.5')
        ibs.db_version_expected = new_version
        ibs.sqldb_fname = new_fname
        ibs.db = sqldbc.SQLDatabaseController(ibs.get_ibsdir(), ibs.sqldb_fname,
                                              text_factory=const.__STR__,
                                              inmemory=False, )
        # Ensure correct schema versions
        _sql_helpers.ensure_correct_version(
            ibs,
            ibs.db,
            ibs.db_version_expected,
            DB_SCHEMA,
            verbose=ut.VERBOSE,
        )
        #print(ibs.sqldbcache_fname)
        #import sys
        #sys.exit(1)

    #@ut.indent_func
    def _init_sqldbcache(ibs):
        """ Need to reinit this sometimes if cache is ever deleted """
        from ibeis.control import _sql_helpers
        from ibeis.control import SQLDatabaseControl as sqldbc
        from ibeis.control import DBCACHE_SCHEMA
        # IBEIS SQL Features & Chips database
        ibs.dbcache_version_expected = '1.0.4'
        # Test a new schema if developer
        new_version, new_fname = sqldbc.dev_test_new_schema_version(
            ibs.get_dbname(), ibs.get_cachedir(),
            ibs.sqldbcache_fname, ibs.dbcache_version_expected,
            version_next='1.0.4')
        ibs.dbcache_version_expected = new_version
        ibs.sqldbcache_fname = new_fname
        # Create cache sql database
        ibs.dbcache = sqldbc.SQLDatabaseController(
            ibs.get_cachedir(), ibs.sqldbcache_fname,
            text_factory=const.__STR__)
        _sql_helpers.ensure_correct_version(
            ibs,
            ibs.dbcache,
            ibs.dbcache_version_expected,
            DBCACHE_SCHEMA,
            dobackup=False,  # Everything in dbcache can be regenerated.
            verbose=ut.VERBOSE,
        )

    def _close_sqldbcache(ibs):
        ibs.dbcache.close()
        ibs.dbcache = None

    def disconnect_sqldatabase(ibs):
        print('disconnecting from sql database')
        ibs.dbcache.close()
        ibs.dbcache = None
        ibs.db.close()
        ibs.db = None

    @default_decorator
    def clone_handle(ibs, **kwargs):
        ibs2 = IBEISController(dbdir=ibs.get_dbdir(), ensure=False)
        if len(kwargs) > 0:
            ibs2.update_query_cfg(**kwargs)
        #if ibs.qreq is not None:
        #    ibs2._prep_qreq(ibs.qreq.qaids, ibs.qreq.daids)
        return ibs2

    @default_decorator
    def backup_database(ibs):
        from ibeis.control import _sql_helpers
        _sql_helpers.database_backup(ibs.get_ibsdir(), ibs.sqldb_fname,
                                     ibs.backupdir)

    @default_decorator
    def _send_wildbook_request(ibs, wbaddr, payload=None):
        if wbaddr is None:
            return
        try:
            if payload is None:
                response = requests.get(wbaddr)
            else:
                response = requests.post(wbaddr, data=payload)
        # except requests.MissingSchema:
        #     print('[ibs._send_wildbook_request] Invalid URL: %r' % wbaddr)
        #     return None
        except requests.ConnectionError:
            print('[ibs.wb_reqst] Could not connect to Wildbook server at %r' %
                  wbaddr)
            return None
        return response

    @default_decorator
    def _init_dirs(ibs, dbdir=None, dbname='testdb_1',
                   workdir='~/ibeis_workdir', ensure=True):
        """
        Define ibs directories
        """
        PATH_NAMES = const.PATH_NAMES
        REL_PATHS = const.REL_PATHS

        if not ut.QUIET:
            print('[ibs._init_dirs] ibs.dbdir = %r' % dbdir)
        if dbdir is not None:
            workdir, dbname = split(dbdir)
        ibs.workdir  = ut.truepath(workdir)
        ibs.dbname = dbname
        ibs.sqldb_fname = PATH_NAMES.sqldb
        ibs.sqldbcache_fname = PATH_NAMES.sqldbcache

        # Make sure you are not nesting databases
        assert PATH_NAMES._ibsdb != ut.dirsplit(ibs.workdir), \
            'cannot work in _ibsdb internals'
        assert PATH_NAMES._ibsdb != dbname,\
            'cannot create db in _ibsdb internals'
        ibs.dbdir    = join(ibs.workdir, ibs.dbname)
        # All internal paths live in <dbdir>/_ibsdb
        # TODO: constantify these
        # so non controller objects (like in score normalization) have access to
        # these
        ibs._ibsdb      = join(ibs.dbdir, REL_PATHS._ibsdb)
        ibs.trashdir    = join(ibs.dbdir, REL_PATHS.trashdir)
        ibs.cachedir    = join(ibs.dbdir, REL_PATHS.cache)
        ibs.backupdir   = join(ibs.dbdir, REL_PATHS.backups)
        ibs.chipdir     = join(ibs.dbdir, REL_PATHS.chips)
        ibs.imgdir      = join(ibs.dbdir, REL_PATHS.images)
        ibs.treesdir    = join(ibs.dbdir, REL_PATHS.trees)
        ibs.uploadsdir  = join(ibs.dbdir, REL_PATHS.uploads)
        # All computed dirs live in <dbdir>/_ibsdb/_ibeis_cache
        ibs.thumb_dpath = join(ibs.dbdir, REL_PATHS.thumbs)
        ibs.flanndir    = join(ibs.dbdir, REL_PATHS.flann)
        ibs.qresdir     = join(ibs.dbdir, REL_PATHS.qres)
        ibs.bigcachedir = join(ibs.dbdir, REL_PATHS.bigcache)
        ibs.distinctdir = join(ibs.dbdir, REL_PATHS.distinctdir)
        if ensure:
            ibs.ensure_directories()
        assert dbdir is not None, 'must specify database directory'

    def ensure_directories(ibs):
        """
        Makes sure the core directores for the controller exist
        """
        _verbose = ut.VERBOSE
        ut.ensuredir(ibs._ibsdb)
        ut.ensuredir(ibs.cachedir,    verbose=_verbose)
        ut.ensuredir(ibs.backupdir,   verbose=_verbose)
        ut.ensuredir(ibs.workdir,     verbose=_verbose)
        ut.ensuredir(ibs.imgdir,      verbose=_verbose)
        ut.ensuredir(ibs.chipdir,     verbose=_verbose)
        ut.ensuredir(ibs.flanndir,    verbose=_verbose)
        ut.ensuredir(ibs.qresdir,     verbose=_verbose)
        ut.ensuredir(ibs.bigcachedir, verbose=_verbose)
        ut.ensuredir(ibs.thumb_dpath, verbose=_verbose)
        ut.ensuredir(ibs.distinctdir, verbose=_verbose)
        ibs.get_smart_patrol_dir()

    #
    #
    #--------------
    # --- DIRS ----
    #--------------

    @register_api('/api/core/dbname/', methods=['GET'])
    def get_dbname(ibs):
        """
        Returns:
            list_ (list): database name

        RESTful:
            Method: GET
            URL:    /api/core/dbname/
        """
        return ibs.dbname

    def get_logdir(ibs):
        return ut.get_logging_dir(appname='ibeis')

    def get_dbdir(ibs):
        """
        Returns:
            list_ (list): database dir with ibs internal directory """
        #return join(ibs.workdir, ibs.dbname)
        return ibs.dbdir

    def get_db_core_path(ibs):
        """
        Returns:
            path (str): path of the sqlite3 core database file """
        return ibs.db.fpath

    def get_db_cache_path(ibs):
        """
        Returns:
            path (str): path of the sqlite3 cache database file """
        return ibs.dbcache.fpath

    def get_trashdir(ibs):
        return ibs.trashdir

    def get_ibsdir(ibs):
        """
        Returns:
            list_ (list): ibs internal directory """
        return ibs._ibsdb

    def get_chipdir(ibs):
        return ibs.chipdir

    def get_probchip_dir(ibs):
        return join(ibs.get_cachedir(), 'prob_chips')

    def get_fig_dir(ibs):
        """
        Returns:
            list_ (list): ibs internal directory """
        return join(ibs._ibsdb, 'figures')

    def get_imgdir(ibs):
        """
        Returns:
            list_ (list): ibs internal directory """
        return ibs.imgdir

    def get_treesdir(ibs):
        """
        Returns:
            list_ (list): ibs internal directory """
        return ibs.treesdir

    def get_uploadsdir(ibs):
        """
        Returns:
            list_ (list): ibs internal directory """
        return ibs.uploadsdir

    def get_thumbdir(ibs):
        """
        Returns:
            list_ (list): database directory where thumbnails are cached """
        return ibs.thumb_dpath

    def get_workdir(ibs):
        """
        Returns:
            list_ (list): directory where databases are saved to """
        return ibs.workdir

    def get_cachedir(ibs):
        """
        Returns:
            list_ (list): database directory of all cached files """
        return ibs.cachedir

    def get_match_thumbdir(ibs):
        match_thumb_dir = ut.unixjoin(ibs.get_cachedir(), 'match_thumbs')
        ut.ensuredir(match_thumb_dir)
        return match_thumb_dir

    def get_ibeis_resource_dir(ibs):
        """ returns the global resource dir in .config or AppData or whatever """
        resource_dir = sysres.get_ibeis_resource_dir()
        return resource_dir

    def get_global_species_scorenorm_cachedir(ibs, species_text, ensure=True):
        """

        Args:
            species_text (str):
            ensure       (bool):

        Returns:
            str: species_cachedir

        CommandLine:
            python -m ibeis.control.IBEISControl --test-get_global_species_scorenorm_cachedir

        Example:
            >>> # ENABLE_DOCTEST
            >>> from ibeis.control.IBEISControl import *  # NOQA
            >>> import ibeis  # NOQA
            >>> ibs = ibeis.opendb('testdb1')
            >>> species_text = ibeis.const.Species.ZEB_GREVY
            >>> ensure = True
            >>> species_cachedir = ibs.get_global_species_scorenorm_cachedir(species_text, ensure)
            >>> resourcedir = ibs.get_ibeis_resource_dir()
            >>> result = ut.relpath_unix(species_cachedir, resourcedir)
            >>> print(result)
            scorenorm/zebra_grevys

        """
        scorenorm_cachedir = join(ibs.get_ibeis_resource_dir(),
                                  const.PATH_NAMES.scorenormdir)
        species_cachedir = join(scorenorm_cachedir, species_text)
        if ensure:
            ut.ensurepath(scorenorm_cachedir)
            ut.ensuredir(species_cachedir)
        return species_cachedir

    def get_local_species_scorenorm_cachedir(ibs, species_text, ensure=True):
        """
        """
        scorenorm_cachedir = join(ibs.get_cachedir(),
                                  const.PATH_NAMES.scorenormdir)
        species_cachedir = join(scorenorm_cachedir, species_text)
        if ensure:
            ut.ensuredir(scorenorm_cachedir)
            ut.ensuredir(species_cachedir)
        return species_cachedir

    def get_global_distinctiveness_modeldir(ibs, ensure=True):
        """
        Returns:
            global_distinctdir (str): ibs internal directory
        """
        global_distinctdir = sysres.get_global_distinctiveness_modeldir(ensure=ensure)
        return global_distinctdir

    def get_local_distinctiveness_modeldir(ibs):
        """
        Returns:
            distinctdir (str): ibs internal directory """
        return ibs.distinctdir

    def get_detect_modeldir(ibs):
        return join(sysres.get_ibeis_resource_dir(), 'detectmodels')

    def get_detectimg_cachedir(ibs):
        """
        Returns:
            detectimgdir (str): database directory of image resized for
                detections
        """
        return join(ibs.cachedir, const.PATH_NAMES.detectimg)

    def get_flann_cachedir(ibs):
        """
        Returns:
            flanndir (str): database directory where the FLANN KD-Tree is
                stored
        """
        return ibs.flanndir

    def get_qres_cachedir(ibs):
        """
        Returns:
            qresdir (str): database directory where query results are stored """
        return ibs.qresdir

    def get_neighbor_cachedir(ibs):
        neighbor_cachedir = ut.unixjoin(ibs.get_cachedir(), 'neighborcache2')
        return neighbor_cachedir

    def get_big_cachedir(ibs):
        """
        Returns:
            bigcachedir (str): database directory where aggregate results are
                stored
        """
        return ibs.bigcachedir

    def get_smart_patrol_dir(ibs, ensure=True):
        """
        Args:
            ensure (bool):

        Returns:
            str smart_patrol_dpath

        CommandLine:
            python -m ibeis.control.IBEISControl --test-get_smart_patrol_dir

        Example:
            >>> # ENABLE_DOCTEST
            >>> from ibeis.control.IBEISControl import *  # NOQA
            >>> import ibeis
            >>> # build test data
            >>> ibs = ibeis.opendb('testdb1')
            >>> ensure = True
            >>> # execute function
            >>> smart_patrol_dpath = ibs.get_smart_patrol_dir(ensure)
            >>> # verify results
            >>> ut.assertpath(smart_patrol_dpath, verbose=True)
        """
        smart_patrol_dpath = join(ibs.dbdir, const.PATH_NAMES.smartpatrol)
        if ensure:
            ut.ensuredir(smart_patrol_dpath)
        return smart_patrol_dpath

    #
    #
    #------------------
    # --- DETECTION ---
    #------------------

    @default_decorator
    @accessor_decors.getter_1to1
    @register_api('/api/core/detect_random_forest/', methods=['PUT', 'GET'])
    def detect_random_forest(ibs, gid_list, species, **kwargs):
        """
        Runs animal detection in each image. Adds annotations to the database
        as they are found.

        Args:
            gid_list (list): list of image ids to run detection on
            species (str): string text of the species to identify

        Returns:
            aids_list (list): list of lists of annotation ids detected in each
                image

        CommandLine:
            python -m ibeis.control.IBEISControl --test-detect_random_forest --show

        RESTful:
            Method: PUT
            URL:    /api/core/detect_random_forest/

        Example:
            >>> # DISABLE_DOCTEST
            >>> from ibeis.control.IBEISControl import *  # NOQA
            >>> import ibeis
            >>> # build test data
            >>> ibs = ibeis.opendb('testdb1')
            >>> gid_list = ibs.get_valid_gids()[0:2]
            >>> species = ibeis.const.Species.ZEB_PLAIN
            >>> # execute function
            >>> aids_list = ibs.detect_random_forest(gid_list, species)
            >>> # Visualize results
            >>> if ut.show_was_requested():
            >>>     import plottool as pt
            >>>     from ibeis.viz import viz_image
            >>>     for fnum, gid in enumerate(gid_list):
            >>>         viz_image.show_image(ibs, gid, fnum=fnum)
            >>>     pt.show_if_requested()
            >>> # Remove newly detected annotations
            >>> ibs.delete_annots(ut.flatten(aids_list))
        """
        # TODO: Return confidence here as well
        print('[ibs] detecting using random forests')
        from ibeis.model.detect import randomforest  # NOQA
        if isinstance(gid_list, int):
            gid_list = [gid_list]
        print('TYPE:' + str(type(gid_list)))
        print('GID_LIST:' + ut.truncate_str(str(gid_list)))
        detect_gen = randomforest.detect_gid_list_with_species(
            ibs, gid_list, species, **kwargs)
        # ibs.cfg.other_cfg.ensure_attr('detect_add_after', 1)
        # ADD_AFTER_THRESHOLD = ibs.cfg.other_cfg.detect_add_after
        print('TYPE:' + str(type(detect_gen)))
        aids_list = []
        for gid, (gpath, result_list) in zip(gid_list, detect_gen):
            aids = []
            for result in result_list:
                # Ideally, species will come from the detector with confidences
                # that actually mean something
                bbox = (result['xtl'], result['ytl'],
                        result['width'], result['height'])
                (aid,) = ibs.add_annots(
                    [gid], [bbox], notes_list=['rfdetect'],
                    species_list=[species], quiet_delete_thumbs=True,
                    detect_confidence_list=[result['confidence']])
                aids.append(aid)
            aids_list.append(aids)
        return aids_list

    #
    #
    #-----------------------------
    # --- ENCOUNTER CLUSTERING ---
    #-----------------------------

    def _parse_smart_xml(back, xml_path, nTotal, offset=1):
        # Storage for the patrol encounters
        xml_dir, xml_name = split(xml_path)
        encounter_info_list = []
        last_photo_number = None
        last_encounter_info = None
        # Parse the XML file for the information
        patrol_tree = ET.parse(xml_path)
        namespace = '{http://www.smartconservationsoftware.org/xml/1.1/patrol}'
        # Load all waypoint elements
        element = './/%swaypoints' % (namespace, )
        waypoint_list = patrol_tree.findall(element)
        if len(waypoint_list) == 0:
            # raise IOError('There are no observations (waypoints) in this Patrol XML file: %r' % (xml_path, ))
            print('There are no observations (waypoints) in this Patrol XML file: %r' % (xml_path, ))
        for waypoint in waypoint_list:
            # Get the relevant information about the waypoint
            waypoint_id   = int(waypoint.get('id'))
            waypoint_lat  = float(waypoint.get('y'))
            waypoint_lon  = float(waypoint.get('x'))
            waypoint_time = waypoint.get('time')
            waypoint_info = [
                xml_name,
                waypoint_id,
                (waypoint_lat, waypoint_lon),
                waypoint_time,
            ]
            if None in waypoint_info:
                raise IOError('The observation (waypoint) is missing information: %r' % (waypoint_info, ))
            # Get all of the waypoint's observations (we expect only one
            # normally)
            element = './/%sobservations' % (namespace, )
            observation_list = waypoint.findall(element)
            # if len(observation_list) == 0:
            #     raise IOError('There are no observations in this waypoint, waypoint_id: %r' % (waypoint_id, ))
            for observation in observation_list:
                # Filter the observations based on type, we only care
                # about certain types
                categoryKey = observation.attrib['categoryKey']
                if categoryKey.startswith('animals.liveanimals') or categoryKey.startswith('animals.problemanimal'):
                    # Get the photonumber attribute for the waypoint's
                    # observation
                    element = './/%sattributes[@attributeKey="photonumber"]' % (namespace, )
                    photonumber = observation.find(element)
                    if photonumber is not None:
                        element = './/%ssValue' % (namespace, )
                        # Get the value for photonumber
                        sValue  = photonumber.find(element)
                        if sValue is None:
                            raise IOError('The photonumber sValue is missing from photonumber, waypoint_id: %r' % (waypoint_id, ))
                        # Python cast the value
                        try:
                            photo_number = int(float(sValue.text)) - offset
                        except ValueError:
                            # raise IOError('The photonumber sValue is invalid, waypoint_id: %r' % (waypoint_id, ))
                            print('[ibs]     Skipped Invalid Observation with photonumber: %r, waypoint_id: %r' % (sValue.text, waypoint_id, ))
                            continue
                        # Check that the photo_number is within the acceptable bounds
                        if photo_number >= nTotal:
                            raise IOError('The Patrol XML file is looking for images that do not exist (too few images given)')
                        # Keep track of the last waypoint that was processed
                        # becuase we only have photono, which indicates start
                        # indices and doesn't specify the end index.  The
                        # ending index is extracted as the next waypoint's
                        # photonum minus 1.
                        if last_photo_number is not None and last_encounter_info is not None:
                            encounter_info = last_encounter_info + [(last_photo_number, photo_number)]
                            encounter_info_list.append(encounter_info)
                        last_photo_number = photo_number
                        last_encounter_info = waypoint_info
                    else:
                        # raise IOError('The photonumber value is missing from waypoint, waypoint_id: %r' % (waypoint_id, ))
                        print('[ibs]     Skipped Empty Observation with "categoryKey": %r, waypoint_id: %r' % (categoryKey, waypoint_id, ))
                else:
                    print('[ibs]     Skipped Incompatible Observation with "categoryKey": %r, waypoint_id: %r' % (categoryKey, waypoint_id, ))
        # Append the last photo_number
        if last_photo_number is not None and last_encounter_info is not None:
            encounter_info = last_encounter_info + [(last_photo_number, nTotal)]
            encounter_info_list.append(encounter_info)
        return encounter_info_list

    #@ut.indent_func('[ibs.compute_encounters]')
    def compute_encounters_smart(ibs, gid_list, smart_xml_fpath):
        """
        Function to load and process a SMART patrol XML file
        """
        # Get file and copy to ibeis database folder
        xml_dir, xml_name = split(smart_xml_fpath)
        dst_xml_path = join(ibs.get_smart_patrol_dir(), xml_name)
        ut.copy(smart_xml_fpath, dst_xml_path, overwrite=True)
        # Process the XML File
        print('[ibs] Processing Patrol XML file: %r' % (dst_xml_path, ))
        try:
            encounter_info_list = ibs._parse_smart_xml(dst_xml_path, len(gid_list))
        except Exception as e:
            ibs.delete_images(gid_list)
            print('[ibs] ERROR: Parsing Patrol XML file failed, rolling back by deleting %d images...' % (len(gid_list, )))
            raise e
        if len(gid_list) > 0:
            # Sanity check
            assert len(encounter_info_list) > 0, (
                'Trying to added %d images, but the Patrol  XML file has no observations' % (len(gid_list), ))
        # Display the patrol encounters
        for index, encounter_info in enumerate(encounter_info_list):
            smart_xml_fname, smart_waypoint_id, gps, local_time, range_ = encounter_info
            start, end = range_
            gid_list_ = gid_list[start:end]
            print('[ibs]     Found Patrol Encounter: %r' % (encounter_info, ))
            print('[ibs]         GIDs: %r' % (gid_list_, ))
            if len(gid_list_) == 0:
                print('[ibs]         SKIPPING EMPTY ENCOUNTER')
                continue
            # Add the GPS data to the iamges
            gps_list  = [ gps ] * len(gid_list_)
            ibs.set_image_gps(gid_list_, gps_list)
            # Create a new encounter
            enctext = '%s Waypoint %03d' % (xml_name.replace('.xml', ''), index + 1, )
            eid = ibs.add_encounters(enctext)
            # Add images to the encounters
            eid_list = [eid] * len(gid_list_)
            ibs.set_image_eids(gid_list_, eid_list)
            # Set the encounter's smart fields
            ibs.set_encounter_smart_xml_fnames([eid], [smart_xml_fname])
            ibs.set_encounter_smart_waypoint_ids([eid], [smart_waypoint_id])
            # Set the encounter's time based on the images
            unixtime_list = sorted(ibs.get_image_unixtime(gid_list_))
            start_time = unixtime_list[0]
            end_time = unixtime_list[-1]
            ibs.set_encounter_start_time_posix([eid], [start_time])
            ibs.set_encounter_end_time_posix([eid], [end_time])
        # Complete
        print('[ibs] ...Done processing Patrol XML file')

    #@ut.indent_func('[ibs.compute_encounters]')
    def compute_encounters(ibs):
        """
        Clusters ungrouped images into encounters

        CommandLine:
            python -m ibeis.control.IBEISControl --test-compute_encounters

        Example:
            >>> # ENABLE_DOCTEST
            >>> from ibeis.control import *  # NOQA
            >>> import ibeis  # NOQA
            >>> # build test data
            >>> ibs = ibeis.opendb('testdb1')
            >>> ibs.compute_encounters()
            >>> ibs.update_special_encounters()
            >>> # Now we want to remove some images from a non-special encounter
            >>> nonspecial_eids = [i for i in ibs.get_valid_eids() if i not in ibs.get_special_eids()]
            >>> images_to_remove = ibs.get_encounter_gids(nonspecial_eids[0:1])[0][0:1]
            >>> ibs.unrelate_images_and_encounters(images_to_remove,nonspecial_eids[0:1] * len(images_to_remove))
            >>> ibs.update_special_encounters()
            >>> ungr_eid = ibs.get_encounter_eids_from_text(const.UNGROUPED_IMAGES_ENCTEXT)
            >>> ungr_gids = ibs.get_encounter_gids([ungr_eid])[0]
            >>> #Now let's make sure that when we recompute encounters, our non-special eid remains the same
            >>> print('PRE COMPUTE: Encounters are %r' % ibs.get_valid_eids())
            >>> print('Containing: %r' % ibs.get_encounter_gids(ibs.get_valid_eids()))
            >>> ibs.compute_encounters()
            >>> print('COMPUTE: New encounters are %r' % ibs.get_valid_eids())
            >>> print('Containing: %r' % ibs.get_encounter_gids(ibs.get_valid_eids()))
            >>> ibs.update_special_encounters()
            >>> print('UPDATE SPECIAL: New encounters are %r' % ibs.get_valid_eids())
            >>> print('Containing: %r' % ibs.get_encounter_gids(ibs.get_valid_eids()))
            >>> assert(images_to_remove[0] not in ibs.get_encounter_gids(nonspecial_eids[0:1])[0])
        """
        from ibeis.model.preproc import preproc_encounter
        print('[ibs] Computing and adding encounters.')
        #gid_list = ibs.get_valid_gids(require_unixtime=False, reviewed=False)
        # only cluster ungrouped images
        gid_list = ibs.get_ungrouped_gids()
        with ut.Timer('computing encounters'):
            flat_eids, flat_gids = preproc_encounter.ibeis_compute_encounters(
                ibs, gid_list)
        valid_eids = ibs.get_valid_eids()
        eid_offset = 0 if len(valid_eids) == 0 else max(valid_eids)
        # This way we can make sure that manually separated encounters
        flat_eids_offset = [eid + eid_offset for eid in flat_eids]
        # remain untouched, and ensure that new encounters are created
        enctext_list = ['Encounter ' + str(eid) for eid in flat_eids_offset]
        #print('enctext_list: %r; flat_gids: %r' % (enctext_list, flat_gids))
        print('[ibs] Finished computing, about to add encounter.')
        ibs.set_image_enctext(flat_gids, enctext_list)
        # HACK TO UPDATE ENCOUNTER POSIX TIMES
        # CAREFUL THIS BLOWS AWAY SMART DATA
        ibs.update_encounter_info(ibs.get_valid_eids())
        print('[ibs] Finished computing and adding encounters.')

    #
    #
    #-----------------------
    # --- IDENTIFICATION ---
    #-----------------------

    @default_decorator
    @register_api('/api/core/get_current_log_text/', methods=['GET'])
    def get_current_log_text(ibs):
        """

        Example:
            >>> # WEB_DOCTEST
            >>> from ibeis.control.IBEISControl import *  # NOQA
            >>> import time
            >>> import ibeis
            >>> import requests
            >>> # Start up the web instance
            >>> web_instance = ibeis.opendb_in_background(db='testdb1', web=True, browser=False)
            >>> time.sleep(.5)
            >>> baseurl = 'http://127.0.1.1:5000'
            >>> resp = requests.get(baseurl + '/api/core/get_current_log_text/')
            >>> print(resp)
            >>> web_instance.terminate()
            >>> json_dict = resp.json()
            >>> text = json_dict['response']
            >>> print(text)
        """
        text = ut.get_current_log_text()
        return text

    @default_decorator
    @register_api('/api/core/recognition_query_aids/', methods=['GET'])
    def get_recognition_query_aids(ibs, is_known, species=None):
        """
        DEPCIRATE

        RESTful:
            Method: GET
            URL:    /api/core/recognition_query_aids/
        """
        qaid_list = ibs.get_valid_aids(is_known=is_known, species=species)
        return qaid_list

    @register_api('/api/core/query_chips_simple_dict/', methods=['GET'])
    def query_chips_simple_dict(ibs, *args, **kwargs):
        r"""
        Runs query_chips, but returns a json compatible dictionary

        Args:
            same as query_chips

        RESTful:
            Method: GET
            URL:    /api/core/query_chips_simple_dict/

        SeeAlso:
            query_chips

        CommandLine:
            python -m ibeis.control.IBEISControl --test-query_chips_simple_dict:0
            python -m ibeis.control.IBEISControl --test-query_chips_simple_dict:1

            python -m ibeis.control.IBEISControl --test-query_chips_simple_dict:0 --humpbacks

        Example:
            >>> # WEB_DOCTEST
            >>> from ibeis.control.IBEISControl import *  # NOQA
            >>> import ibeis
            >>> ibs = ibeis.opendb(defaultdb='testdb1')
            >>> #qaid = ibs.get_valid_aids()[0:3]
            >>> qaids = ibs.get_valid_aids()
            >>> daids = ibs.get_valid_aids()
            >>> dict_list = ibs.query_chips_simple_dict(qaids, daids, return_cm=True)
            >>> qgids = ibs.get_annot_image_rowids(qaids)
            >>> qnids = ibs.get_annot_name_rowids(qaids)
            >>> for dict_, qgid, qnid in zip(dict_list, qgids, qnids):
            >>>     dict_['qgid'] = qgid
            >>>     dict_['qnid'] = qnid
            >>>     dict_['dgid_list'] = ibs.get_annot_image_rowids(dict_['daid_list'])
            >>>     dict_['dnid_list'] = ibs.get_annot_name_rowids(dict_['daid_list'])
            >>>     dict_['dgname_list'] = ibs.get_image_gnames(dict_['dgid_list'])
            >>>     dict_['qgname'] = ibs.get_image_gnames(dict_['qgid'])
            >>> result  = ut.list_str(dict_list, nl=2, precision=2, hack_liststr=True)
            >>> result = result.replace('u\'', '"').replace('\'', '"')
            >>> print(result)

        Example:
            >>> # WEB_DOCTEST
            >>> from ibeis.control.IBEISControl import *  # NOQA
            >>> import time
            >>> import ibeis
            >>> import requests
            >>> # Start up the web instance
            >>> web_instance = ibeis.opendb_in_background(db='testdb1', web=True, browser=False)
            >>> time.sleep(.5)
            >>> baseurl = 'http://127.0.1.1:5000'
            >>> data = dict(qaid_list=[1])
            >>> resp = requests.get(baseurl + '/api/core/query_chips_simple_dict/', data=data)
            >>> print(resp)
            >>> web_instance.terminate()
            >>> json_dict = resp.json()
            >>> cmdict_list = json_dict['response']
            >>> assert 'score_list' in cmdict_list[0]

        """
        kwargs['return_cm_simple_dict'] = True
        return ibs.query_chips(*args, **kwargs)

    @register_api('/api/core/query_chips_dict/', methods=['GET'])
    def query_chips_dict(ibs, *args, **kwargs):
        """
        Runs query_chips, but returns a json compatible dictionary

        RESTful:
            Method: GET
            URL:    /api/core/query_chips_dict/
        """
        kwargs['return_cm_dict'] = True
        return ibs.query_chips(*args, **kwargs)

    @register_api('/api/core/query_chips/', methods=['GET'])
    def query_chips(ibs, qaid_list=None,
                    daid_list=None,
                    cfgdict=None,
                    use_cache=None,
                    use_bigcache=None,
                    qreq_=None,
                    return_request=False,
                    verbose=pipeline.VERB_PIPELINE,
                    save_qcache=None,
                    prog_hook=None,
                    return_cm=False,
                    return_cm_dict=False,
                    return_cm_simple_dict=False,
                    ):
        r"""
        Submits a query request to the hotspotter recognition pipeline. Returns
        a list of QueryResult objects.

        Note:
            In the future the QueryResult objects will be replaced by ChipMatch
            objects

        Args:
            qaid_list (list): a list of annotation ids to be submitted as queries
            daid_list (list): a list of annotation ids used as the database that will be searched
            cfgdict (dict): dictionary of configuration options used to create a new QueryRequest if not already specified
            use_cache (bool):
            use_bigcache (bool):
            qreq_ (QueryRequest): optional, a QueryRequest object that overrides all previous settings
            return_request (bool): returns the request which will be created if one is not already specified
            verbose (bool): default=False, turns on verbose printing
            return_cm (bool): default=False, if true converts QueryResult objects into serializable ChipMatch2 objects (in the future this will be defaulted to True)

        Returns:
            list: a list of QueryResult objects containing the matching annotations, scores, and feature matches

        Returns(2):
            tuple: (qres_list, qreq_) - a list of query results and optionally the QueryRequest object used

        CommandLine:
            python -m ibeis.control.IBEISControl --test-query_chips

        RESTful:
            Method: PUT
            URL:    /api/core/query_chips/

        Example:
            >>> # SLOW_DOCTEST
            >>> from ibeis.control.IBEISControl import *  # NOQA
            >>> import ibeis
            >>> ibs = ibeis.opendb('testdb1')
            >>> qaids = ibs.get_valid_aids()[0:1]
            >>> qres = ibs.query_chips(qaids)[0]
            >>> assert qres.qaid == qaids[0]

        """
        # The qaid and daid objects are allowed to be None if qreq_ is
        # specified
        if qaid_list is None:
            qaid_list = qreq_.get_external_qaids()
        if daid_list is None:
            if qreq_ is not None:
                daid_list = qreq_.get_external_daids()
            else:
                daid_list = ibs.get_valid_aids()

        qaid_list, was_scalar = ut.wrap_iterable(qaid_list)

        # Wrapped call to the main entrypoint in the API to the hotspotter
        # pipeline
        _res = ibs._query_chips4(
            qaid_list, daid_list, cfgdict=cfgdict, use_cache=use_cache,
            use_bigcache=use_bigcache, qreq_=qreq_,
            return_request=return_request, verbose=verbose,
            save_qcache=save_qcache,
            prog_hook=prog_hook)
        if return_request:
            qaid2_qres, qreq_ = _res
        else:
            qaid2_qres = _res

        # Return a list of query results instead of that awful dictionary
        # that will be depricated in future version of hotspotter.
        qres_list = [qaid2_qres[qaid] for qaid in qaid_list]

        if return_cm or return_cm_dict or return_cm_simple_dict:
            from ibeis.model.hots import chip_match
            cm_list = [
                chip_match.ChipMatch2.from_qres(qres) for qres in qres_list]
            # Convert to cm_list
            if return_cm_simple_dict:
                qres_list = [cm.to_simple_dict() for cm in cm_list]
            elif return_cm_dict:
                qres_list = [cm.to_dict() for cm in cm_list]
            else:
                qres_list = cm_list

        if was_scalar:
            # hack for scalar input
            assert len(qres_list) == 1
            qres_list = qres_list[0]

        if return_request:
            return qres_list, qreq_
        else:
            return qres_list

    @register_api('/api/core/query_chips4/', methods=['PUT'])
    def _query_chips4(ibs, qaid_list, daid_list,
                      use_cache=None,
                      use_bigcache=None,
                      return_request=False,
                      cfgdict=None,
                      qreq_=None,
                      verbose=pipeline.VERB_PIPELINE,
                      save_qcache=None,
                      prog_hook=None):
        """
        submits a query request
        main entrypoint in the IBIES API to the hotspotter pipeline

        CommandLine:
            python -m ibeis.control.IBEISControl --test-_query_chips4

        RESTful:
            Method: PUT
            URL:    /api/core/query_chips4/

        Example:
            >>> # SLOW_DOCTEST
            >>> #from ibeis.all_imports import *  # NOQA
            >>> from ibeis.control.IBEISControl import *  # NOQA
            >>> qaid_list = [1]
            >>> daid_list = [1, 2, 3, 4, 5]
            >>> ibs = ibeis.test_main(db='testdb1')
            >>> qres = ibs._query_chips4(qaid_list, daid_list, use_cache=False)[1]

        #>>> qreq_ = mc4.get_ibeis_query_request(ibs, qaid_list, daid_list)
        #>>> qreq_.load_indexer()
        #>>> qreq_.load_query_vectors()
        #>>> qreq = ibs.qreq
        """
        from ibeis.model.hots import match_chips4 as mc4
        # Check fo empty queries
        try:
            assert len(daid_list) > 0, 'there are no database chips'
            assert len(qaid_list) > 0, 'there are no query chips'
        except AssertionError as ex:
            ut.printex(ex, 'Impossible query request', iswarning=True,
                       keys=['qaid_list', 'daid_list'])
            if ut.SUPER_STRICT:
                raise
            qaid2_qres = {qaid: None for qaid in qaid_list}
            if return_request:
                return qaid2_qres, qreq_
            else:
                return qaid2_qres

        # Check for consistency
        if qreq_ is not None:
            ut.assert_lists_eq(
                qreq_.get_external_qaids(), qaid_list,
                'qaids do not agree with qreq_', verbose=True)
            ut.assert_lists_eq(
                qreq_.get_external_daids(), daid_list,
                'daids do not agree with qreq_', verbose=True)

        # Send query to hotspotter (runs the query)
        _res = mc4.submit_query_request(
            ibs,  qaid_list, daid_list, use_cache, use_bigcache,
            return_request=return_request, cfgdict=cfgdict, qreq_=qreq_,
            verbose=verbose, save_qcache=save_qcache, prog_hook=prog_hook)

        if return_request:
            qaid2_qres, qreq_ = _res
            return qaid2_qres, qreq_
        else:
            qaid2_qres = _res
            return qaid2_qres

    #_query_chips = _query_chips3
    _query_chips = _query_chips4

    @default_decorator
    @register_api('/api/core/query_encounter/', methods=['PUT'])
    def query_encounter(ibs, qaid_list, eid, **kwargs):
        """
        _query_chips wrapper

        RESTful:
            Method: PUT
            URL:    /api/core/query_encounter/
        """
        daid_list = ibs.get_encounter_aids(eid)  # encounter database chips
        qaid2_qres = ibs._query_chips4(qaid_list, daid_list, **kwargs)
        # HACK IN ENCOUNTER INFO
        for qres in six.itervalues(qaid2_qres):
            qres.eid = eid
        return qaid2_qres

    @default_decorator
    @register_api('/api/core/query_exemplars/', methods=['PUT'])
    def query_exemplars(ibs, qaid_list, **kwargs):
        """
        Queries vs the exemplars

        RESTful:
            Method: PUT
            URL:    /api/core/query_exemplars/
        """
        daid_list = ibs.get_valid_aids(is_exemplar=True)
        assert len(daid_list) > 0, 'there are no exemplars'
        return ibs._query_chips4(qaid_list, daid_list, **kwargs)

    @default_decorator
    @register_api('/api/core/query_all/', methods=['PUT'])
    def query_all(ibs, qaid_list, **kwargs):
        """
        DEPRICATED

        Queries vs all valid chip ~~the exemplars~~

        RESTful:
            Method: PUT
            URL:    /api/core/query_all/
        """
        daid_list = ibs.get_valid_aids()
        qaid2_qres = ibs._query_chips4(qaid_list, daid_list, **kwargs)
        return qaid2_qres

    @default_decorator
    @register_api('/api/core/has_species_detector/', methods=['GET'])
    def has_species_detector(ibs, species_text):
        """
        TODO: extend to use non-constant species

        RESTful:
            Method: GET
            URL:    /api/core/has_species_detector/
        """
        # FIXME: infer this
        return species_text in const.SPECIES_WITH_DETECTORS

    @default_decorator
    @register_api('/api/core/species_with_detectors/', methods=['GET'])
    def get_species_with_detectors(ibs):
        """
        RESTful:
            Method: GET
            URL:    /api/core/species_with_detectors/
        """
        # FIXME: infer this
        return const.SPECIES_WITH_DETECTORS

if __name__ == '__main__':
    """
    Issue when running on windows:
    python ibeis/control/IBEISControl.py
    python -m ibeis.control.IBEISControl --verbose --very-verbose --veryverbose --nodyn --quietclass

    CommandLine:
        python -m ibeis.control.IBEISControl
        python -m ibeis.control.IBEISControl --allexamples
        python -m ibeis.control.IBEISControl --allexamples --noface --nosrc
    """
    #from ibeis.control import IBEISControl
    import multiprocessing
    multiprocessing.freeze_support()  # for win32
    import utool as ut  # NOQA
    ut.doctest_funcs()


import json

from braces.views import LoginRequiredMixin, AjaxResponseMixin, JSONResponseMixin
from django.contrib.auth import authenticate, login as auth_login, logout as auth_logout
from django.core.urlresolvers import reverse
from django.db.models.fields import AutoField
from django.shortcuts import redirect
from django.views import generic as cbv

from apps.api.client_service import api_service
from apps.office.office.forms import UserForm

class LoginView(AjaxResponseMixin, JSONResponseMixin, cbv.RedirectView):

    def get_redirect_url(self, *args, **kwargs):
        if self.request.user.is_authenticated():
            return reverse('office:main')
        else:
            return '/'

    def post_ajax(self, request, *args, **kwargs):
        hash = request.POST.get('hash')
        auth = False
        result = {}
        if hash:
            user_id = api_service.get_user_id_by_hash(hash)
            if user_id:
                user = authenticate(user_id)
                if user:
                    autofield = AutoField()
                    autofield.attname = 'pk'
                    user._meta.pk = autofield
                    auth_login(self.request, user)
                    auth = True
                else:
                    auth_logout(self.request)
        if auth:
            result = {
                'url': reverse('office:main')
            }
        return self.render_json_response(json.dumps(result))

login_view = LoginView.as_view()

class LogoutView(cbv.RedirectView):
    url = '/'
    permanent = False

    def get_redirect_url(self, *args, **kwargs):
        return self.url

    def get(self, request, *args, **kwargs):
        auth_logout(request)
        return super(LogoutView, self).get(request, *args, **kwargs)

logout_view = LogoutView.as_view()

class UpdateUserView(LoginRequiredMixin, cbv.View):
    http_method_names = ['post']

    def post(self, request, *args, **kwargs):
        form = UserForm(request.POST)
        if form.is_valid():
            form.save()
        return redirect(reverse('office:main'))

update_user_view = UpdateUserView.as_view()


	#!/usr/bin/python

'''
Author : Marie Degen

Cluster the transcrits from a genes family
Example : python -W ignore clustering.py similarityScores/FAM86_score.csv 0.5

'''

import sys
import argparse
import os
import numpy as np
import pandas as pd
import pickle
import json

def build_arg_parser(path):
	parser = argparse.ArgumentParser(description="Label exons")
	parser.add_argument('-s', '--save_path', default =  path + '/clusters/')
	parser.add_argument('-i', '--inputFile',  required=True)
	parser.add_argument('-m', '--minSimilarity',  default = 0.5)
	return parser

'''
Return true if the transcrits are from the same gene, false otherwise
'''
def checkSameGene(transcrit1, transcrit2):
	if len(transcrit1.split("-")) < 2  and len(transcrit2.split("-")) < 2:
		return True if transcrit1.split("_")[1] == transcrit2.split("_")[1] else False
	elif len(transcrit1.split("-")) > 2  and len(transcrit2.split("-")) < 2:
		for gene in transcrit1.split("-"):
			if gene.split("_")[1] == transcrit2.split("_")[1]:
				return True
			else:
				continue
	elif len(transcrit1.split("-")) < 2  and len(transcrit2.split("-")) > 2:
		for gene in transcrit2.split("-"):
			if gene.split("_")[1] == transcrit1.split("_")[1]:
				return True
			else:
				continue
	elif len(transcrit1.split("-")) > 2  and len(transcrit2.split("-")) > 2:
		for gene1, gene2 in zip(transcrit2.split("-"), transcrit1.split("-")):
			if gene1.split("_")[1] == gene2.split("_")[1]:
				return True
			else:
				continue

'''
Return true if the transcrits are the same
'''
def checkSameTranscrit(transcrit1, transcrit2):
	return True if transcrit1.split("_")[0] == transcrit2.split("_")[0] else False

'''
Retrieve the names of the header
'''
def retrieveName(matrix):
	names = []
	for val in range(len(matrix)):
		names.append(matrix[val][0].replace("\n",""))
	return names

'''
Reconstruct the matrix from the file, return a matrix with only the values and a dataframe with the header
'''
def retrieveValues(matrixValues, size):
	values = np.eye(size, size+1)

	for i, v in enumerate(matrixValues):
		for j, o in enumerate(v):
			if type(o) == str  :
				continue
			else:
				values[i][j] = o

	values = np.delete(values,0,1)
	return values

'''
Return the maximum value of similarity found in the matrix, the two transcrits corresponding
'''
def returnMaxValueSimilarity(matrix, names, bannedGroup):
	maxValue = 0
	tr1 = ""
	tr2 = ""

	for i in range(len(matrix)):
		for j in range(len(matrix)):
			if names[i] == names[j]:
				continue
			else:
				if checkSameGene(names[i],names[j]):
					continue
				elif names[i].replace("\n","") in bannedGroup or names[j].replace("\n","") in bannedGroup:
					continue
				else:
					if matrix[i][j] > maxValue:
						tr1 = names[i]
						tr2 = names[j]
						maxValue = matrix[i][j]
	return maxValue, tr1, tr2

'''
Return the matrix to join to the bigger matrix, the matrix to join has the average calculated in it
'''
def matrixDistCalculate(matrix, transcrit1, transcrit2, bannedGroup, names):
	matrixToJoin = []
	for k in range(len(matrix)):
		nbvalue = 0
		moyenne = 0
		summoyenne =0
		for j in range(len(matrix)):
			if checkSameGene(names[k],transcrit2) or checkSameGene(names[k],transcrit1) :
				if len(transcrit2.split("-")) > 2 or len(transcrit1.split("-")) > 2:
					if checkSameGene(names[k],transcrit2) or checkSameGene(names[k],transcrit2):
						continue
					elif checkSameGene(names[k],transcrit1) or checkSameGene(names[k],transcrit1):
						continue
				else:
					continue
			elif names[k] in bannedGroup:
				continue
			else:
				summoyenne += matrix[k][j]
				nbvalue += 1
			try:
				moyenne = summoyenne / nbvalue
			except:
				moyenne = 0.0
		matrixToJoin.append(float("{0:.2f}".format(moyenne)))
	matrixToJoin.append(1.0)
	return matrixToJoin

def main_cluster(save_path, inputFile, minSimilarity):
	size = 0
	matrixValues = []
	cluster = []
	bannedGroup = []
	initGroup = []

	matrixScore = pd.read_csv(inputFile)
	size = matrixScore.shape[0]
	matrixValues = matrixScore.values

	names = retrieveName(matrixValues)
	values = retrieveValues(matrixValues, size)
	initGroup = names[:]
	namefiletmp = inputFile.split("/")[-1]
	namefile = namefiletmp.split(".")[0]
	#familyName = namefiletmp.split("_")[1]
	#namefile = namefile + "_" + familyName + "_clusters.out"
	namefile = namefile + "_clusters.out"
	completeName = os.path.join(save_path, namefile)
	outputfile = open(completeName, "w")

	while True:
		maximum, transcrit1, transcrit2 = returnMaxValueSimilarity(values, names, bannedGroup)
		if maximum <= float(minSimilarity):
			break
		else:
			#delete the transrit from the finalgroup
			initGroup.remove(transcrit1.replace("\n",""))
			initGroup.remove(transcrit2.replace("\n",""))

			name_cluster = "(" + transcrit1.replace("\n","") + "-" + transcrit2.replace("\n","") + ")"

			cluster.append([transcrit1.replace("\n",""), transcrit2.replace("\n",""), maximum])
			bannedGroup.append(transcrit1.replace("\n",""))
			bannedGroup.append(transcrit2.replace("\n",""))
			names.append(name_cluster)
			initGroup.append(name_cluster)

			matrixToJoin = matrixDistCalculate(values, transcrit1, transcrit2, bannedGroup, names)

			#resize the matrix with the new values
			n = values.shape[0]
			w = np.eye(n+1, n+1)
			w[:-1,:-1] = values
			w[-1:] = matrixToJoin
			w[:,-1] = matrixToJoin
			values = w

	outputfile.write(str(initGroup))
	return completeName
	#pickle.dump(initGroup, outputfile)
	#json.dump(initGroup, outputfile)

def main_cluster_aux(inputFile, minSimilarity):
	path =  sys.path[0]
	save_path  = path + "/clusters/"
	return main_cluster(save_path, inputFile, minSimilarity)

if __name__ == "__main__":
	path =  sys.path[0]
	parser = build_arg_parser(path)
	arg = parser.parse_args()
	save_path  = arg.save_path
	inputFile = arg.inputFile
	minSimilarity = arg.minSimilarity
	main_cluster(save_path, inputFile, minSimilarity)


#Q2

from threading import *
from time import *
def display():
    print('Value of 1-10 in interval of 1 second')
    for i in range(1,11):
        sleep(1)
        print(i)

t=Thread(target=display)
t.start()


import importlib
import os
import keras.backend as K

import cv2
import numpy as np
from keras.engine.saving import load_model

from classifier.classifier_utils import get_yolo3_output, draw_boxes
from classifier.model import create_classifier_model
from yolo import RegressionLayer

if __name__ == '__main__':
    K.set_learning_phase(0)

    config = importlib.import_module('config').config
    os.environ['CUDA_VISIBLE_DEVICES'] = config['predict']['cuda_devices']
    batch_size = config['predict']['classifier_batch_size']

    infer_model = load_model(config['predict']['infer_model_path'], custom_objects={'RegressionLayer': RegressionLayer})
    C2 = infer_model.get_layer('leaky_10').output
    C3 = infer_model.get_layer('leaky_35').output
    C4 = infer_model.get_layer('leaky_60').output
    C5 = infer_model.get_layer('leaky_78').output

    classifier_model = create_classifier_model(
        image_shape=(config['predict']['classifier_net_size'], config['predict']['classifier_net_size'], 3),
        classes_count=len(config['categories']))
    assert os.path.exists(config['predict']['classifier_weights_path'])
    classifier_model.load_weights(config['predict']['classifier_weights_path'])
    classifier_model.summary()

    video_reader = cv2.VideoCapture(config['predict']['videofile_path'])

    frames_processed = 0
    cv2.namedWindow('image', cv2.WINDOW_KEEPRATIO)
    while True:
        read, image = video_reader.read()
        if not read:
            break
        orig_image = image.copy()

        frames_processed += 1
        if frames_processed % config['predict']['every_nth_frame'] != 0:
            continue

        boxes, crops = get_yolo3_output(
            infer_model, image,
            config['predict']['infer_net_size'],
            config['predict']['classifier_net_size'],
            config['predict']['confidence'],
            config['predict']['nms_threshold'])
        if len(crops) == 0:
            cv2.imshow('image', image)
            cv2.waitKey(1)
            continue

        h, w, _ = image.shape

        classes = np.zeros((len(crops), len(config['categories'])))
        for i in range(len(crops) // batch_size):
            out = classifier_model.predict_on_batch(crops[i * batch_size: (i + 1) * batch_size, ...])
            for j in range(len(out)):
                classes[i + j] = out[j]

        boxes = [[b.xmin, b.ymin, b.xmax, b.ymax, np.max(classes[i]), 1 if np.isclose(classes[i, np.argmax(classes[i])], 0.) else np.argmax(classes[i])]
                 for i, b in enumerate(boxes)]
        image = draw_boxes(image, boxes, config['categories'])
        cv2.imshow('image', image)
        key = cv2.waitKeyEx(1)
        if key == 27:  # esc
            break

        # process_nth_frame
        elif key == 81 or key == 113:  # q
            config['predict']['every_nth_frame'] = config['predict']['every_nth_frame'] \
                if config['predict']['every_nth_frame'] <= 1 else config['predict']['every_nth_frame'] - 1
        elif key == 69 or key == 101:  # e
            config['predict']['every_nth_frame'] = config['predict']['every_nth_frame'] \
                if config['predict']['every_nth_frame'] >= 100 else config['predict']['every_nth_frame'] + 1

        # net_size
        elif key == 90 or key == 122:  # z
            config['predict']['infer_net_size'] = config['predict']['infer_net_size'] \
                if config['predict']['infer_net_size'] <= 64 else config['predict']['infer_net_size'] - 32
        elif key == 67 or key == 99:  # c
            config['predict']['infer_net_size'] = config['predict']['infer_net_size'] \
                if config['predict']['infer_net_size'] >= 1600 else config['predict']['infer_net_size'] + 32

        # obj_thresh
        elif key == 64 or key == 97:  # a
            config['predict']['confidence'] = config['predict']['confidence'] \
                if config['predict']['confidence'] <= 0.1 else config['predict']['confidence'] - 0.05
        elif key == 68 or key == 100:  # d
            config['predict']['confidence'] = config['predict']['confidence'] \
                if config['predict']['confidence'] >= 0.95 else config['predict']['confidence'] + 0.05

        print('\rProcessed {} frame: net_size[{}]; obj_thresh[{:.2f}]; nms_thresh[{:.2f}]; process_nth_frame[{}]'.format(
            frames_processed, config['predict']['infer_net_size'], config['predict']['confidence'],
            config['predict']['nms_threshold'], config['predict']['every_nth_frame']
        ), end='')
    cv2.destroyAllWindows()


# JOSE ALFREDO ROMERO GONZALEZ
# 18/11/2020
palabra = input("Ingrese una palabra: \n")
for i in range(10):
    print(palabra)

# SOLUCION DE https://aprendeconalf.es/
word = input("Introduce una palabra: ")
for i in range(10):
    print(word)


#!/usr/bin/env python
# (c) John Strickler 2018

p2 = ['Tom', 'Bombadil', 45]  # list

p3 = {'Tom', 'Bombadil', 45}  # set

p4 = {'firstname': 'Tom', 'lastname': 'Bombadil', 'age': 45} # dict

person = 'Tom', 'Bombadil', 45 # tuple
print(type(person))
print(person[0], person[1])

first_name, last_name, age = person # iterable unpacking
print(first_name, last_name)

def get_person():
    return 'Tom', 'Bombadil', 45

first_name, last_name, age = get_person()
print(first_name, last_name)

person = get_person()
print(person[0], person[1])

people = [
    ('Melinda', 'Gates', 'Gates Foundation'),
    ('Steve', 'Jobs', 'Apple', 'NeXT'),
    ('Larry', 'Wall', 'Perl'),
    ('Paul', 'Allen', 'Microsoft'),
    ('Larry', 'Ellison', 'Oracle'),
    ('Bill', 'Gates', 'Microsoft'),
    ('Mark', 'Zuckerberg', 'Facebook'),
    ('Sergey','Brin', 'Google', 'ABC'),
    ('Larry', 'Page', 'Google'),
    ('Linus', 'Torvalds', 'Linux', 'git'),
    ('John', 'Strickler'),
]

for first_name, last_name, *product in people:
    print(first_name, last_name, product)
print('-' * 60)

values = ['a', 'b', 'c', 'd', 'e', 'f']

x, y, *z = values
print(x, y, z)

x, *y, z = values
print(x, y, z)

*x, y, z = values
print(x, y, z)


from bs4 import BeautifulSoup
import requests

f = open('output.txt','w')
# 2827 pages
for i in range(2501, 2827):
    url = "http://allrecipes.com/recipes/?sort=Title&page=" + str(i)
    print(i)
    r = requests.get(url)
    soup = BeautifulSoup(r.content, "lxml")
    links = soup.find_all('article', {'class': "fixed-recipe-card"})
    for ele in links:
        f.write(ele.a["href"] + '\n')

f.close()


# 文件读写
import random as rd

f = open("D:/udacity/python/first-script.py")
data = f.read()
# print(data)
f.close()

# with 开头语法，执行完代码块，文件自动关闭
lines = []
with open('D:/udacity/python/first-script.py') as f:
    line = f.readline()
    print('first line:', line)
    for line in f:
        lines.append(line)
# print(lines)
print(rd.choice(lines))
print(rd.sample(lines,2))


#calss header
class _GOVERN():
	def __init__(self,):
		self.name = "GOVERN"
		self.definitions = [u'to control and direct the public business of a country, city, group of people, etc.: ', u'to have a controlling influence on something: ']

		self.parents = []
		self.childen = []
		self.properties = []
		self.jsondata = {}

		self.specie = 'verbs'

	def run(self, obj1 = [], obj2 = []):
		return self.jsondata


"""AbayTracker URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/3.0/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.urls import path
from . import views
from .dash_apps import dash_abay
#from .dash_apps import dash_uber
#from .dash_apps import dash_tutorial

app_name = "AbayDashboard"

urlpatterns = [
    path('', views.dashboard, name="dashboard"),  # This runs the working dashboard without django_dash
    path('dash_django', views.dash_django, name="dash_django"),  # This runs the dashboard with dash
    path('dash', views.dash, name="plotly"),
    path('dash_tutorial', views.dash_tutorial, name="plotly_tutorial"),
    path("login", views.login_request, name="login"),
]


import cv2
import numpy as np
import os
import pickle
from time import time
from scipy.ndimage import gaussian_laplace as LoG
import matplotlib.pyplot as plt

query_path = "./train/query"
image_path = "./images"
k = 2**(1/2)
thresh = 0.03
blobs = []
show = True
dk = 0
train = False

if train :
    for i in os.listdir(image_path):
        print(dk)
        dk+=1
        result =[]
        blobs  =[]
        img = cv2.imread(image_path+'/'+i)
        w = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        w = w/255
        # x = np.array(img, dtype=np.float64)/255
        for j in range(1,11):
            sigma = (k**j)
            g = cv2.GaussianBlur(w,(3,3),sigma)
            result.append(np.square(cv2.Laplacian(g,cv2.CV_64F)))

        result = np.stack(result)
        for j in range(np.shape(w)[0]):
            for h in range(np.shape(w)[1]):
                arr = result[:, j:j+1, h:h+1]
                # print(np.argmax(arr))
                if  np.amax(arr) > thresh :
                    z,x,y = np.unravel_index(np.argmax(arr),arr.shape)
                    blobs.append((x+j-1,y+h-1,(k**z)))

            blobs  = list(set(blobs))
            pickle.dump(blobs,open('./log/'+i.split('.')[0]+'.p','wb'))

if show :
    for i in os.listdir('./log'):
        # i = 'all_souls_000001.p'
        blob = pickle.load(open('./log/'+i , 'rb'))
        if np.shape(blob)[0] > 2000 :
            blobs = []
            for j in range(0,np.shape(blob)[0],4):
                blobs.append(blob[j])
        else :
            blobs = blob
        img = cv2.imread(image_path+'/'+i.split('.')[0]+'.jpg')
        # img/= 255
        print(np.shape(blobs))
        fig, ax = plt.subplots()
        ax.imshow(img)
        for blob in blobs:
            ax.add_patch(plt.Circle((blob[1], blob[0]), blob[2]*1.414, color='red', linewidth=0.5, fill=False))
        ax.plot()
        plt.show()


import re
import tqdm
import glob
import pickle
import concurrent.futures

from esra.transformers.entity_merging import coreference_handler
from esra.transformers.post_processing import Post_processor
from esra.transformers.generic_remover import remove_generic
from esra.transformers.conjunction_relations_merger import merge_conjuction_relation
from esra.transformers.entity_merging import duplicate_entity_handler
from esra.transformers.abbreviation_splitter import abbreviation_split
from esra.transformers.cycle_counter import CycleCounter

list_data = []
for filename in glob.glob('./data/pickle/kaggle_arxiv_*_*.pickle'):
    print(filename)
    with open(filename, 'rb') as f:
        list_data += pickle.load(f)
print(len(list_data))

list_valid_data = []
list_invalid_data = []
pp = Post_processor()
cc = CycleCounter(threshold=3)

def validate(data):
    # NOTE: Pass data id for identification purpose
    meta = {'id': data['id']}

    data = coreference_handler(data)
    data = abbreviation_split(data)
    data = pp.post_processing(data)
    data = remove_generic(data)
    data = merge_conjuction_relation(data)

    # NOTE: Hot fix name of entities
    for en in data['entities']:
        # en[1] = en[1].lower()
        en[1] = re.sub(r'( \- |\- | \-)', '-', en[1]) # connect hyphens
        en[1] = re.sub(r'[\(\)]', '', en[1]) # remove single sided parenthesis
        en[1] = re.sub(r' +', ' ', en[1]) # remove double spaces
        en[1] = en[1].strip()

    data = duplicate_entity_handler(data)
    data = cc.drop_self_loops(data)

    # NOTE: Add data id here
    data.update(meta)
    return data

for i in range(0, 24000, 1000):

    print(i)
    collection = list_data[i:i+1000]

    with concurrent.futures.ProcessPoolExecutor(max_workers=40) as executor:
        r = [executor.submit(validate, data) for data in collection]
        results = [f.result() for f in concurrent.futures.as_completed(r)]

    for data in results:
        if cc.cyclic_validate(data):
            list_invalid_data.append(data)
        else:
            list_valid_data.append(data)

with open('./data/pickle/kaggle_arxiv_cleaned.pickle','wb') as f:
    pickle.dump(list_valid_data, f)
    print("Done!")


#!/usr/bin/env python

import json
from datetime import datetime
import paho.mqtt.client as mqtt

from flask import Flask, jsonify, make_response, request
from random import randint

globalRed = 0
globalGreen = 0
globalBlue = 0
globalLastCalled = None
globalLastCalledApi = None

app = Flask(__name__)

def setTimestamp() :
	global globalLastCalled
	globalLastCalled = datetime.now()

def switchOn() :
	red = randint(10, 255)
	green = randint(10, 255)
	blue = randint(10, 255)
	client = mqtt.Client()
	client.connect("192.168.1.100",1883,60)
	client.publish("cmnd/tasmota_D9264E/Dimmer", '35')
	client.disconnect()
	publish(red,green,blue)

def switchOff() :
	global  globalBlue, globalGreen, globalRed
	globalRed = 0
	globalGreen = 0
	globalBlue = 0
	client = mqtt.Client()
	client.connect("192.168.1.100",1883,60)
	client.publish("cmnd/tasmota_D9264E/POWER", 'OFF')
	client.disconnect()

def publish(red,green,blue) :
	payload = str(red) + ',' + str(green) + ',' + str(blue)
	client = mqtt.Client()
	client.connect("192.168.1.100",1883,60)
	client.publish("cmnd/tasmota_D9264E/Color2", payload)
	client.disconnect()

# API Initialization
@app.route('/api/on', methods=['GET'])
def apiOn() :
	global globalLastCalledApi
	globalLastCalledApi = '/api/on'
	content = request.json
	print(content)
	switchOff()
	switchOn()
	setTimestamp()
	return jsonify({})

@app.route('/api/off', methods=['GET'])
def apiOff() :
	global crntColors, globalLastCalledApi
	globalLastCalledApi = '/api/off'
	crntColors = None
	content = request.json
	print(content)
	switchOff()
	setTimestamp()
	return jsonify({})

@app.route('/api/switch', methods=['POST'])
def apiSwitch() :
	global globalLastCalledApi
	globalLastCalledApi = '/api/switch'
	switchOff()
	content = request.json
	print(content)
	red = content.get('red', '')
	green = content.get('green', '')
	blue = content.get('blue', '')
	setTimestamp()
	publish(red,green,blue)
	return make_response(jsonify())

@app.route('/api/status', methods=['GET'])
def apiStatus() :
	global globalBlue, globalGreen, globalRed, globalLastCalled, globalLastCalledApi
	return jsonify({ 'red': globalRed, 'green': globalGreen, 'blue': globalBlue, 'lastCalled': globalLastCalled, 'lastCalledApi': globalLastCalledApi })

@app.errorhandler(404)
def not_found(error):
	return make_response(jsonify({'error': 'Not found'}), 404)

if __name__ == '__main__':
	app.run(host='0.0.0.0', debug=False)


import hashlib as hl  # Used for checksums

# Read the container file
input_file = open('transfer_test.part', 'rb')
checksum = input_file.read(40)  # Read the checksum bytes
data_header = input_file.read(100)  # Read the NMEA Message
data = input_file.read()  # Read the image data
input_file.close()

# Checksum the file
checksum_generated = hl.sha1(data_header+data).hexdigest()

# Verify file integrity
if checksum.decode('ascii') != checksum_generated:
    raise Exception('Checksum no match')

# Write the NEMA string with padding
header_file = open('files/transfer_header.txt', 'wb')
header_file.write(data_header)
header_file.close()

# Write the image data
data_file = open('files/transfer_data.ARW', 'wb')
data_file.write(data)
data_file.close()


import string

class ConfigDictionary(dict):
    '''
    A dictionary that allows self reference by using {} syntax for string objects.

    Basic idea can be found here:
    http://stackoverflow.com/questions/3738381/what-do-i-do-when-i-need-a-self-referential-dictionary
    '''

    def __init__(self, *args, **kw):

        super(ConfigDictionary, self).__init__(*args, **kw)
        self.itemlist = super(ConfigDictionary, self).keys()
        self.fmt = string.Formatter()

    def __getitem__(self, item):

        result = dict.__getitem__(self, item)
        result_type = type(result)

        if result_type is str:
            return self.fmt.vformat(result, {}, self)
        elif result_type is ConfigDictionary:
            result.update(self)
            return result
        elif result_type is list:
            for index, each in enumerate(result):
                if type(each) is str and each is not '{' and each is not '}':
                    result[index] = self.fmt.vformat(each, {}, self)
            return result
        else:
            return result


import time

import numpy as np
from numpy.random import rand
from scipy.constants import epsilon_0
from numpy import pi

from xobjects.context import ContextCpu, ContextCupy, ContextPyopencl
from xfields import TriLinearInterpolatedFieldMap

context = ContextCpu()
context = ContextCupy(default_block_size=256)
context = ContextPyopencl()

print(repr(context))

import matplotlib.pyplot as plt
plt.close('all')

center_xyz = np.array([0.1, 0.2, -0.3])
radius = .5
n_part_cube = 10000000

x_cube = radius*(2. * rand(n_part_cube) - 1.) + center_xyz[0]
y_cube = radius*(2. * rand(n_part_cube) - 1.) + center_xyz[1]
z_cube = radius*(2. * rand(n_part_cube) - 1.) + center_xyz[2]

mask_sphere = ((x_cube - center_xyz[0])**2
             + (y_cube - center_xyz[1])**2
             + (z_cube - center_xyz[2])**2) < radius**2
x = x_cube[mask_sphere]
y = y_cube[mask_sphere]
z = z_cube[mask_sphere]
pweights = 1. + 0*x

# PIC
x_lim = (-1.1, 1.)
y_lim = (-1.2, 1.)
z_lim = (-1.3, 1.)

nx = 128
ny = 128
nz = 200

theta_test = 70. * np.pi/360
phi_test = 300 * np.pi/360.
r_test = 2.5*radius*np.linspace(-1, 1., 1000)

x0_test = r_test * np.sin(theta_test) * np.cos(phi_test)
y0_test = r_test * np.sin(theta_test) * np.sin(phi_test)
z0_test = r_test * np.cos(theta_test)

x_test = center_xyz[0] + x0_test
y_test = center_xyz[1] + y0_test
z_test = center_xyz[2] + z0_test

# Moving particles and test coordinates to GPU (if needed)
np2platf = context.nparray_to_context_array
x_dev = np2platf(x)
y_dev = np2platf(y)
z_dev = np2platf(z)
pweights_dev = np2platf(pweights)
x_test_dev = np2platf(x_test)
y_test_dev = np2platf(y_test)
z_test_dev = np2platf(z_test)

###############
# Actual test #
###############

# Build fieldmap object
fmap = TriLinearInterpolatedFieldMap(
        _context=context,
        x_range=x_lim, nx=nx,
        y_range=y_lim, ny=ny,
        z_range=z_lim, nz=nz,
        solver='FFTSolver3D')

# Compute potential
n_rep = 4
for _ in range(n_rep):
    t1 = time.time()
    fmap.update_from_particles(x_p=x_dev, y_p=y_dev, z_p=z_dev,
            ncharges_p=pweights_dev, q0_coulomb=1.)

    # Check on the x axis
    (rho_test_dev, phi_test_dev, dx_test_dev, dy_test_dev,
            dz_test_dev) = fmap.get_values_at_points(
                x=x_test_dev, y=y_test_dev, z=z_test_dev)
    context.synchronize()
    t2 = time.time()
    print(f'Time: {t2-t1:.2e}')

# Copy back for plotting
platf2np = context.nparray_from_context_array
rho_test =  platf2np(rho_test_dev)
phi_test = platf2np(phi_test_dev)
dx_test = platf2np(dx_test_dev)
dy_test = platf2np(dy_test_dev)
dz_test = platf2np(dz_test_dev)

# pickle for other tests
import pickle
# with open('picsphere.pkl', 'wb') as fid:
#     pickle.dump({
#         'fmap': fmap,
#         'x_test': x_test,
#         'y_test': y_test,
#         'z_test': z_test,
#         }, fid)

####################
# Plots and checks #
####################

ex_test = -dx_test
ey_test = -dy_test
ez_test = -dz_test

plt.figure(100)
plt.plot(r_test, rho_test)
plt.axhline(y=len(x)/(4/3*np.pi*radius**3))

plt.figure(101)
plt.plot(r_test, phi_test)

e_ref = len(x)/(4*pi*epsilon_0) * (
        r_test/radius**3*(np.abs(r_test)<radius)
      + np.sign(r_test)/r_test**2*(np.abs(r_test)>=radius))
plt.figure(102)
plt.plot(r_test, ex_test*x0_test/r_test
        + ey_test*y0_test/r_test + ez_test*z0_test/r_test)
plt.plot(r_test, e_ref)
plt.grid(True)

# Check integral
int_rho = np.sum(platf2np(fmap.rho))*fmap.dx*fmap.dy*fmap.dz
assert np.isclose(int_rho, len(x))

fig1 = plt.figure(1)
ax1 = fig1.add_subplot(111)
ax1.pcolormesh(fmap.x_grid, fmap.y_grid,
        np.sum(platf2np(fmap.rho), axis=2).T, shading='gouraud')
ax1.set_aspect('equal')
ax1.add_patch(plt.Circle((center_xyz[0], center_xyz[1]), radius,
                         color='w', fill=False))
ax1.set_xlabel('x')
ax1.set_ylabel('y')

fig2 = plt.figure(2)
ax2 = fig2.add_subplot(111)
ax2.pcolormesh(fmap.y_grid, fmap.z_grid,
        np.sum(platf2np(fmap.rho), axis=0).T, shading='gouraud')
ax2.set_aspect('equal')
ax2.add_patch(plt.Circle((center_xyz[1], center_xyz[2]), radius,
                         color='w', fill=False))
ax2.set_xlabel('y')
ax2.set_ylabel('z')
plt.show()


import sys
from PyQt5 import QtCore

def timerEvent():
    global time
    time = time.addSecs(-1)
    print(time.toString("hh:mm:ss"))

app = QtCore.QCoreApplication(sys.argv)

timer = QtCore.QTimer()
time = QtCore.QTime(0,0,10)
timer.timeout.connect(timerEvent)
timer.start(1000)

sys.exit(app.exec_()) #파이참에서 실행시키면 카운트다운이 되긴하는데 이걸 UI랑 합치는 법을 모르겠음


from decimal import Decimal
d = Decimal(10)
print(d)
print(d + 20)

0.1 * 3 == 0.3
Decimal('0.1') * 3 == Decimal('0.3')

print(d.sqrt())

class Prism:
    def __init__(self, width, height, depth):
        self.width = width
        self.height = height
        self.depth = depth

    def content(self):
        return self.width * self.height * self.depth

p1 = Prism(10, 20, 30)
p1.content()

p2 = Prism(50, 60, 70)
p2.content()

p1.height
p2.height

p1.height = 50
p1.content()

p = Prism(10, 20, 30)
p.width
p.depth = "30"
p.content()

# class Cube(Prism):
#     def __init__(self, length):
#         self.width = self.height = self.depth = length

class Cube(Prism):
    def __init__(self, length):
        super().__init__(length, length, length)

c = Cube(20)
c.content()

class Prism:
    def __init__(self, width, height, depth, unit='cm'):
        self.width = width
        self.height = height
        self.depth = depth
        self.unit = unit

    def content(self):
        return self.width * self.height * self.depth

    def unit_content(self):
        return(self.content()) + self.unit

class Cube(Prism):
    def __init__(self, length):
        super().__init__(length, length, length)

c = Cube(10)
c.unit_content()

class Klass:
    __slots__ = ['a', 'b']
    def __init__(self):
        self.a = 1

i = Klass()
i.a

i.b = 2
i.b

i.c = 3

class Prop:
    def __init__(self):
        self.__x = 0

    def getx(self):
        return self.__x

    def setx(self, x):
        self.__x = x

    x = property(getx, setx)

i = Prop()
i.x

i.x = 10
i.x

i._Prop__x

from strdict import StrDict
d = StrDict()
d["spam"] = 1
d["spam"]
d[1] = 1
d.keys()


#!/usr/bin/env python3.6

import feedparser as fp
import time
from datetime import datetime, timedelta
import pytz
import sys
from string import Template

with open(sys.argv[1], "r") as f:
    subscriptions = f.readlines()

# Date and time setup. I only want posts from 10PM the day before
utc = pytz.utc
homeTZ = pytz.timezone('Europe/London')

current_time = datetime.now(homeTZ)

start = (current_time - timedelta(hours=24)).replace(hour=22, minute=0, second=0, microsecond=0)
start = start.astimezone(utc)

# Collect all of today's posts and put them in a list of tuples.
blogs = []
try:
    for s in subscriptions:
        f = fp.parse(s)
        try:
            blog = f['feed']['title']
        except KeyError:
            continue

        posts = []
        for e in f['entries']:
            try:
                when = e['updated_parsed']
            except KeyError:
                when = e['published_parsed']
            when =  utc.localize(datetime.fromtimestamp(time.mktime(when)))
            if when > start:
                title = e['title']
                link = e['link']
                posts.append((title, link))

        if posts:
            blogs.append((blog, posts))
except Exception as e:
    print(f'''

    <title>Oh dear</title>

There was an error {e}

''')
    exit()

blogs.sort()
# Turn them into an HTML list.
listTemplate = '''
{0}

        {1}

'''

listItemTemplate = '''

{0}

'''

ulitems = []
for b in blogs:
    litems = []
    for p in b[1]:
        litems.append(listItemTemplate.format(*p))

    ulitems.append(listTemplate.format(b[0], '\n'.join(litems)))

if not ulitems:
    ulitems.append("
No news yet...
")

# Print the HTMl.
with open(sys.argv[2], "r") as f:
    template = Template(f.read())

print(template.substitute(content="\n".join(ulitems)))


"""
PyTorch Video Dataset Class for loading videos using PyTorch
Dataloader. This Dataset assumes that video files are Preprocessed
 by being trimmed over time and resizing the frames.

Mohsen Fayyaz __ Sensifai Vision Group
http://www.Sensifai.com

If you find this code useful, please star the repository.
"""

from __future__ import print_function, division
import cv2
import os
import torch
import numpy as np
import pickle
from torch.utils.data import Dataset, DataLoader
from torchvision import transforms, utils

class RandomCrop(object):
    """Crop randomly the frames in a clip.

	Args:
		output_size (tuple or int): Desired output size. If int, square crop
			is made.
	"""

    def __init__(self, output_size):
        assert isinstance(output_size, (int, tuple))
        if isinstance(output_size, int):
            self.output_size = (output_size, output_size)
        else:
            assert len(output_size) == 2
            self.output_size = output_size

    def __call__(self, clip):

        h, w = clip.size()[2:]
        new_h, new_w = self.output_size

        top = np.random.randint(0, h - new_h)
        left = np.random.randint(0, w - new_w)

        clip = clip[:, :, top: top + new_h,
               left: left + new_w]

        return clip

class videoDataset(Dataset):
    """Dataset Class for Loading Video"""

    def __init__(self, clipsListFile, rootDir, channels, timeDepth, xSize, ySize, mean, transform=None):
        """
		Args:
			clipsList (string): Path to the clipsList file with labels.
			rootDir (string): Directory with all the videoes.
			transform (callable, optional): Optional transform to be applied
				on a sample.
			channels: Number of channels of frames
			timeDepth: Number of frames to be loaded in a sample
			xSize, ySize: Dimensions of the frames
			mean: Mean valuse of the training set videos over each channel
		"""
        with open(clipsListFile, "rb") as fp:   # Unpickling
            clipsList = pickle.load(fp)

        self.clipsList = clipsList
        self.rootDir = rootDir
        self.channels = channels
        self.timeDepth = timeDepth
        self.xSize = xSize
        self.ySize = ySize
        self.mean = mean
        self.transform = transform

    def __len__(self):
        return len(self.clipsList)

    def readVideo(self, videoFile):
        # Open the video file
        cap = cv2.VideoCapture(videoFile)
        # nFrames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
        frames = torch.FloatTensor(self.channels, self.timeDepth, self.xSize, self.ySize)
        failedClip = False
        for f in range(self.timeDepth):

            ret, frame = cap.read()
            if ret:
                frame = torch.from_numpy(frame)
                # HWC2CHW
                frame = frame.permute(2, 0, 1)
                frames[:, f, :, :] = frame

            else:
                print("Skipped!")
                failedClip = True
                break

        for c in range(3):
            frames[c] -= self.mean[c]
        frames /= 255
        return frames, failedClip

    def __getitem__(self, idx):

        videoFile = os.path.join(self.rootDir, self.clipsList[idx][0])
        clip, failedClip = self.readVideo(videoFile)
        if self.transform:
            clip = self.transform(clip)
        #sample = {'clip': clip, 'label': self.clipsList[idx][1], 'failedClip': failedClip}
        sample = {'clip': clip}

        return sample