# -*- Mode: Python -*-
# vi:si:et:sw=4:sts=4:ts=4
import sqlite3
import operator
import types

from zope.interface import implements
from twisted.enterprise import adbapi
from twisted.spread import pb
from twisted.internet import reactor
from twisted.python import log as twisted_log

from feat.common import (log, text_helper, error_handler, defer,
                         formatable, enum, decorator, time, manhole,
                         fiber, signal, )
from feat.agencies import common
from feat.common.serialization import banana
from feat.extern.log import log as flulog

from feat.interface.journal import *
from feat.interface.serialization import *
from feat.agencies.interface import *
from feat.interface.log import *

class State(enum.Enum):

    '''
    disconnected - there is no connection to database
    connected - connection is ready, entries can be insterted
    '''
    (disconnected, connected, ) = range(2)

class EntriesCache(object):
    '''
    Helper class storing the data and giving the back in transactional way.
    '''

    def __init__(self):
        self._cache = list()
        self._fetched = None

    def append(self, entry):
        self._cache.append(entry)

    def fetch(self):
        '''
        Gives all the data it has stored, and remembers what it has given.
        Later we need to call commit() to actually remove the data from the
        cache.
        '''
        if self._fetched is not None:
            raise RuntimeError('fetch() was called but the previous one has '
                               'not yet been applied. Not supported')
        if self._cache:
            self._fetched = len(self._cache)
        return self._cache[0:self._fetched]

    def commit(self):
        '''
        Actually remove data returned by fetch() from the cache.
        '''
        if self._fetched is None:
            raise RuntimeError('commit() was called but nothing was fetched')
        self._cache = self._cache[self._fetched:]
        self._fetched = None

    def rollback(self):
        if self._fetched is None:
            raise RuntimeError('rollback() was called but nothing was fetched')
        self._fetched = None

    def is_locked(self):
        '''
        Tells if we are currently in the locked state (need commit or rollback)
        '''
        return self._fetched is not None

    def __len__(self):
        return len(self._cache)

@decorator.parametrized_function
def in_state(func, *states):

    def wrapper(self, *args, **kwargs):
        d = defer.succeed(None)
        if not self._cmp_state(states):
            d.addCallback(defer.drop_param, self.wait_for_state, *states)
        d.addCallback(defer.drop_param, func, self, *args, **kwargs)
        return d

    return wrapper

class Journaler(log.Logger, common.StateMachineMixin):
    implements(IJournaler, ILogKeeper)

    log_category = 'journaler'

    _error_handler = error_handler

    # FIXME: at some point switch to False and remove this attribute
    should_keep_on_logging_to_flulog = True

    def __init__(self, logger):
        log.Logger.__init__(self, self)

        common.StateMachineMixin.__init__(self, State.disconnected)
        self._writer = None
        self._flush_task = None
        self._cache = EntriesCache()
        self._notifier = defer.Notifier()

    def configure_with(self, writer):
        self._ensure_state(State.disconnected)
        twisted_log.addObserver(self.on_twisted_log)
        self._writer = IJournalWriter(writer)
        self._set_state(State.connected)
        self._schedule_flush()

    def close(self, flush_writer=True):

        def set_disconnected():
            self._writer = None
            self._set_state(State.disconnected)

        try:
            twisted_log.removeObserver(self.on_twisted_log)
        except ValueError:
            # it should be safe to call close() multiple times,
            # in this case we are not registered as the observer anymore
            pass

        d = self._close_writer(flush_writer)
        d.addCallback(defer.drop_param, set_disconnected)
        return d

    ### IJournaler ###

    def get_connection(self, externalizer):
        externalizer = IExternalizer(externalizer)
        instance = JournalerConnection(self, externalizer)
        return instance

    def prepare_record(self):
        return Record(self)

    @in_state(State.connected)
    def get_histories(self):
        return self._writer.get_histories()

    @in_state(State.connected)
    def get_entries(self, history):
        return self._writer.get_entries(history)

    def insert_entry(self, **data):
        self._cache.append(data)
        self._schedule_flush()
        return self._notifier.wait('flush')

    @in_state(State.connected)
    def get_filename(self):
        return self._writer.get_filename()

    def is_idle(self):
        if len(self._cache) > 0:
            return False
        if self._writer:
            return self._writer.is_idle()
        return True

    ### ILogObserver provider ###

    def on_twisted_log(self, event_dict):
        edm = event_dict['message']
        if not edm:
            if event_dict['isError'] and 'failure' in event_dict:
                fail = event_dict['failure']
                self.error("A twisted traceback occurred. Exception: %r.",
                           fail.value)
                if flulog.getCategoryLevel("twisted") < flulog.WARN:
                    self.debug(
                        "Run with debug level >= 2 to see the traceback.")
                else:
                    self.error("%s", fail.getTraceback())

    ### ILogKeeper Methods ###

    def do_log(self, level, object, category, format, args,
               depth=-1, file_path=None, line_num=None):
        level = int(level)
        if category is None:
            category = 'feat'
        if level > flulog.getCategoryLevel(category):
            return

        if file_path is None and line_num is None:
            file_path, line_num = flulog.getFileLine(where=-depth-2)

        if args:
            message = format % args
        else:
            message = str(format)

        data = dict(
            entry_type='log',
            level=level,
            log_name=object,
            category=category,
            file_path=file_path,
            line_num=line_num,
            message=message)
        self.insert_entry(**data)

        if self.should_keep_on_logging_to_flulog:
            flulog.doLog(level, object, category, format, args,
                         where=depth, filePath=file_path, line=line_num)

    ### private ###

    def _schedule_flush(self):
        if self._flush_task is None:
            self._flush_task = time.callLater(0, self._flush)

    @in_state(State.connected)
    def _flush(self):
        entries = self._cache.fetch()
        if entries:
            d = self._writer.insert_entries(entries)
            d.addCallbacks(defer.drop_param, self._flush_error,
                           callbackArgs=(self._flush_complete, ))
            return d
        else:
            self._flush_complete()

    def _flush_complete(self):
        if self._cache.is_locked():
            self._cache.commit()
        self._flush_task = None
        self._notifier.callback('flush', None)
        if len(self._cache) > 0:
            self._schedule_flush()

    def _flush_error(self, fail):
        self._cache.rollback()
        fail.raiseException()

    def _close_writer(self, flush_writer=True):
        d = defer.succeed(None)
        if self._writer:
            d.addCallback(defer.drop_param, self._writer.close,
                          flush=flush_writer)
        return d

class BrokerProxyWriter(log.Logger, common.StateMachineMixin):
    implements(IJournalWriter)

    _error_handler = error_handler

    def __init__(self, broker):
        '''
        @param encoding: Optional encoding to be used for blob fields.
        @type encoding: Should be a valid parameter for str.encode() method.
        @param filename: File to use for entries. Defaults to :memory:
        @param logger: ILogger to use
        '''
        log.Logger.__init__(self, broker)
        common.StateMachineMixin.__init__(self, State.disconnected)

        self._broker = broker
        self._set_writer(None)
        self._cache = EntriesCache()
        self._semaphore = defer.DeferredSemaphore(1)

    def initiate(self):
        d = self._broker.get_journal_writer()
        d.addCallback(self._set_writer)
        d.addCallback(defer.drop_param, self._set_state, State.connected)
        return d

    def close(self, flush=True):
        d = defer.succeed(None)
        if flush:
            d.addCallback(defer.drop_param, self._flush_next)
        d.addCallback(defer.drop_param, self._set_state, State.disconnected)
        d.addCallback(defer.drop_param, self._set_writer, None)
        return d

    @in_state(State.connected)
    def get_histories(self):
        return self._writer.callRemote('get_histories')

    @in_state(State.connected)
    def get_entries(self, history):
        return self._writer.callRemote('get_entries', history)

    def insert_entries(self, entries):
        for data in entries:
            self._cache.append(data)
        return self._flush_next()

    @in_state(State.connected)
    def get_filename(self):
        return self._writer.callRemote('get_filename')

    def is_idle(self):
        if len(self._cache) > 0:
            return False
        return True

    ### private ###

    @in_state(State.connected)
    def _flush_next(self):
        if len(self._cache) == 0:
            return defer.succeed(None)
        else:
            d = self._semaphore.run(self._push_entries)
            d.addCallback(defer.drop_param, self._flush_next)
            return d

    def _push_entries(self):
        entries = self._cache.fetch()
        if entries:
            try:
                d = self._writer.callRemote('insert_entries', entries)
                d.addCallbacks(defer.drop_param, defer.drop_param,
                               callbackArgs=(self._cache.commit, ),
                               errbackArgs=(self._cache.rollback, ))
                return d
            except pb.DeadReferenceError:
                # for some reason callRemote raises this error
                # instead of giving failed Deferred
                self._cache.rollback()

    def _set_writer(self, writer):
        self._writer = writer
        if isinstance(self._writer, pb.RemoteReference):
            self._writer.notifyOnDisconnect(self._on_disconnect)

    def _on_disconnect(self, writer):
        self._set_state(State.disconnected)
        self._set_writer(None)

class SqliteWriter(log.Logger, log.LogProxy, common.StateMachineMixin,
                   manhole.Manhole):
    implements(IJournalWriter)

    _error_handler = error_handler

    def __init__(self, logger, filename=":memory:", encoding=None,
                 on_rotate=None):
        '''
        @param encoding: Optional encoding to be used for blob fields.
        @type encoding: Should be a valid parameter for str.encode() method.
        @param filename: File to use for entries. Defaults to :memory:
        @param logger: ILogger to use
        '''
        log.Logger.__init__(self, logger)
        log.LogProxy.__init__(self, logger)
        common.StateMachineMixin.__init__(self, State.disconnected)

        self._encoding = encoding
        self._db = None
        self._filename = filename
        self._reset_history_id_cache()
        self._cache = EntriesCache()
        # the semaphore is used to always have at most running
        # .perform_instert() method
        self._semaphore = defer.DeferredSemaphore(1)

        self._sighup_installed = False

        self._on_rotate_cb = on_rotate

    def initiate(self):
        self._db = adbapi.ConnectionPool('sqlite3', self._filename,
                                         cp_min=1, cp_max=1, cp_noisy=True,
                                         check_same_thread=False,
                                         timeout=10)
        self._install_sighup()
        return self._check_schema()

    ### IJournalWriter ###

    def close(self, flush=True):
        d = defer.succeed(None)
        if self._cmp_state(State.disconnected):
            return d
        if flush:
            d.addCallback(defer.drop_param, self._flush_next)
        d.addCallback(defer.drop_param, self._db.close)
        d.addCallback(defer.drop_param, self._uninstall_sighup)
        d.addCallback(defer.drop_param, self._set_state,
                      State.disconnected)
        return d

    @manhole.expose()
    @in_state(State.connected)
    def get_histories(self):
        return History.fetch(self._db)

    @manhole.expose()
    @in_state(State.connected)
    def get_entries(self, history, start_date=0, limit=None):
        '''
        Returns a list of journal entries  for the given history_id.
        '''
        if not isinstance(history, History):
            raise AttributeError(
                'First paremeter is expected to be History instance, got %r'
                % history)

        command = text_helper.format_block("""
        SELECT histories.agent_id,
               histories.instance_id,
               entries.journal_id,
               entries.function_id,
               entries.fiber_id,
               entries.fiber_depth,
               entries.args,
               entries.kwargs,
               entries.side_effects,
               entries.result,
               entries.timestamp
          FROM entries
          LEFT JOIN histories ON histories.id = entries.history_id
          WHERE entries.history_id = ?""")
        if start_date:
            command += " AND entries.timestamp >= %s" % (start_date, )
        command += " ORDER BY entries.rowid ASC"
        if limit:
            command += " LIMIT %s" % (limit, )
        d = self._db.runQuery(command, (history.history_id, ))
        d.addCallback(self._decode)
        return d

    @in_state(State.connected)
    def get_log_entries(self, start_date=None, end_date=None, filters=list()):
        '''
        @param start_date: epoch time to start search
        @param end_date: epoch time to end search
        @param filters: list of dictionaries with the following keys:
                        level - mandatory, display entries with lvl <= level
                        category - optional, limit to log_category
                        name - optional, limit to log_name
                        Leaving optional fields blank will match all the
                        entries. The entries in this list are combined with
                        OR operator.
        '''
        query = text_helper.format_block('''
        SELECT logs.message,
               logs.level,
               logs.category,
               logs.log_name,
               logs.file_path,
               logs.line_num,
               logs.timestamp
        FROM logs
        WHERE 1
        ''')
        query = self._add_timestamp_condition_sql(query, start_date, end_date)

        def transform_filter(filter):
            level = filter.get('level', None)
            category = filter.get('category', None)
            name = filter.get('name', None)
            if level is None:
                raise AttributeError("level is mandatory parameter.")
            resp = "(logs.level <= %d" % (int(level), )
            if category is not None:
                resp += " AND logs.category == '%s'" % (category, )
            if name is not None:
                resp += " AND logs.log_name == '%s'" % (name, )
            resp += ')'
            return resp

        filter_strings = map(transform_filter, filters)
        if filter_strings:
            query += " AND (%s)\n" % (' OR '.join(filter_strings), )
        d = self._db.runQuery(query)
        d.addCallback(self._decode)
        return d

    @in_state(State.connected)
    def get_log_categories(self, start_date=None, end_date=None):
        '''
        @param start_date: epoch time to start search
        @param end_date: epoch time to end search
        '''
        query = text_helper.format_block('''
        SELECT DISTINCT logs.category
        FROM logs
        WHERE 1
        ''')
        query = self._add_timestamp_condition_sql(query, start_date, end_date)
        d = self._db.runQuery(query)

        def unpack(res):
            return map(operator.itemgetter(0), res)

        d.addCallback(unpack)
        return d

    def get_log_names(self, category, start_date=None, end_date=None):
        '''
        Fetches log names for the given category.
        @param start_date: epoch time to start search
        @param end_date: epoch time to end search
        '''
        query = text_helper.format_block('''
        SELECT DISTINCT logs.log_name
        FROM logs
        WHERE category = ?
        ''')
        query = self._add_timestamp_condition_sql(query, start_date, end_date)
        d = self._db.runQuery(query, (category, ))

        def unpack(res):
            return map(operator.itemgetter(0), res)

        d.addCallback(unpack)
        return d

    def get_log_time_boundaries(self):
        '''
        @returns: a tuple of log entry timestaps (first, last) or None
        '''
        query = text_helper.format_block('''
        SELECT min(logs.timestamp),
               max(logs.timestamp)
        FROM logs''')

        def unpack(res):
            if res:
                return res[0]

        d = self._db.runQuery(query)
        d.addCallback(unpack)
        return d

    @manhole.expose()
    def insert_entries(self, entries):
        for data in entries:
            self._cache.append(data)
        return self._flush_next()

    @manhole.expose()
    def get_filename(self):
        return self._filename

    def is_idle(self):
        if len(self._cache) > 0:
            return False
        return True

    ### Private ###

    def _add_timestamp_condition_sql(self, query, start_date, end_date):
        if start_date is not None:
            query += "  AND logs.timestamp >= %d\n" % (int(start_date), )
        if end_date is not None:
            query += "  AND logs.timestamp <= %d\n" % (int(end_date), )
        return query

    def _reset_history_id_cache(self):
        # (agent_id, instance_id, ) -> history_id
        self._history_id_cache = dict()

    def _sighup_handler(self, signum, frame):
        self.log("Received SIGHUP, reopening the journal.")
        self.close()
        self.initiate()
        if callable(self._on_rotate_cb):
            self._on_rotate_cb()

    def _install_sighup(self):
        if self._sighup_installed:
            return
        if self._filename == ':memory:':
            return
        self.log('Installing SIGHUP handler.')
        handler = signal.signal(signal.SIGHUP, self._sighup_handler)
        self._sighup_installed = True

    def _uninstall_sighup(self):
        if not self._sighup_installed:
            return

        try:
            signal.unregister(signal.SIGHUP, self._sighup_handler)
            self.log("Uninstalled SIGHUP handler.")
        except ValueError:
            self.warning("Unregistering of sighup failed. Straaange!")
        self._sighup_installed = False

    def _decode(self, entries):
        '''
        Takes the list of rows returned by sqlite.
        Returns rows in readable format.
        '''

        def decode_blobs(row):
            row = list(row)
            for index, value in zip(range(len(row)), row):
                if isinstance(value, types.BufferType):
                    value = str(value)
                    if self._encoding:
                        value = value.decode(self._encoding)
                    row[index] = value
            return row

        return map(decode_blobs, entries)

    def _encode(self, data):
        result = dict()

        if data['entry_type'] == 'journal':
            to_copy = ('fiber_depth', 'instance_id', 'entry_type')
            to_decode = ('agent_id', 'function_id', 'fiber_id', )
            to_blob = ('journal_id', 'args', 'kwargs', 'side_effects',
                       'result', )
        elif data['entry_type'] == 'log':
            to_copy = ('level', 'log_name', 'category', 'line_num',
                       'entry_type')
            to_decode = ('file_path', )
            to_blob = ('message', )
        else:
            raise RuntimeError('Unknown entry type %r' % data['entry_type'])

        # just copy, caring open escapes
        for key in to_copy:
            result[key] = data[key]

        for key in to_decode:
            if data[key] is None:
                data[key] = ""
            result[key] = data[key].decode("utf-8")

        # encode the blobs
        for key in to_blob:
            safe = data[key]
            if self._encoding:
                safe = safe.encode(self._encoding)
            result[key] = sqlite3.Binary(safe)

        return result

    def _check_schema(self):
        d = self._db.runQuery(
            'SELECT value FROM metadata WHERE name = "encoding"')
        d.addCallbacks(self._got_encoding, self._create_schema)
        return d

    def _got_encoding(self, res):
        encoding = res[0][0]
        if encoding == 'None':
            encoding = None
        if self._encoding is not None and encoding != self._encoding:
            self.warning("Journaler created with encoding %r but the one "
                         "loaded from existing database is %r. Using "
                         "the value of: %r",
                         self._encoding, encoding, encoding)
        self._encoding = encoding
        self._initiated_ok()

    def _create_schema(self, fail):
        fail.trap(sqlite3.OperationalError)
        commands = [
            text_helper.format_block("""
            CREATE TABLE entries (
              history_id INTEGER NOT NULL,
              journal_id BLOB,
              function_id VARCHAR(200),
              fiber_id VARCHAR(36),
              fiber_depth INTEGER,
              args BLOB,
              kwargs BLOB,
              side_effects BLOB,
              result BLOB,
              timestamp INTEGER
            )
            """),
            text_helper.format_block("""
            CREATE TABLE logs (
              message BLOB,
              level INTEGER,
              category VARCHAR(36),
              log_name VARCHAR(36),
              file_path VARCHAR(200),
              line_num INTEGER,
              timestamp INTEGER
            )
            """),
            text_helper.format_block("""
            CREATE TABLE histories (
              id INTEGER PRIMARY KEY AUTOINCREMENT,
              agent_id VARCHAR(36),
              instance_id INTEGER
            )
            """),
            text_helper.format_block("""
            CREATE TABLE metadata (
              name VARCHAR(100),
              value VARCHAR(100)
            )
            """),
            text_helper.format_block("""
            CREATE INDEX history_idx ON entries(history_id)
            """),
            text_helper.format_block("""
            CREATE INDEX instance_idx ON histories(agent_id, instance_id)
            """)]

        def run_all(connection, commands):
            for command in commands:
                self.log('Executing command:\n %s', command)
                connection.execute(command)

        insert_meta = "INSERT INTO metadata VALUES('%s', '%s')"
        commands += [insert_meta % (u'encoding', self._encoding, )]

        self._reset_history_id_cache()
        # insert_history = "INSERT INTO histories VALUES(%d, '%s', %d)"
        # for (a_id, i_id), h_id in self._history_id_cache.iteritems():
        #     commands += [insert_history % (h_id, a_id, i_id)]

        d = self._db.runWithConnection(run_all, commands)
        d.addCallbacks(self._initiated_ok, self._error_handler)
        return d

    def _initiated_ok(self, *_):
        self.log('Journaler initiated correctly for the filename %r',
                 self._filename)
        self._set_state(State.connected)
        return self._flush_next()

    def _perform_inserts(self, cache):

        def do_insert_entry(connection, history_id, data):
            command = text_helper.format_block("""
            INSERT INTO entries VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?,
                                        strftime('%s', 'now'))
            """)
            connection.execute(
                command, (history_id,
                          data['journal_id'], data['function_id'],
                          data['fiber_id'], data['fiber_depth'],
                          data['args'], data['kwargs'],
                          data['side_effects'], data['result'], ))

        def do_insert_log(connection, data):
            command = text_helper.format_block("""
            INSERT INTO logs VALUES (?, ?, ?, ?, ?, ?, strftime('%s', 'now'))
            """)
            connection.execute(
                command, (data['message'], int(data['level']),
                          data['category'], data['log_name'],
                          data['file_path'], data['line_num']))

        def transaction(connection, cache):
            entries = cache.fetch()
            if not entries:
                return
            try:
                entries = map(self._encode, entries)
                for data in entries:
                    if data['entry_type'] == 'journal':
                        history_id = self._get_history_id(
                            connection, data['agent_id'], data['instance_id'])
                        do_insert_entry(connection, history_id, data)
                    elif data['entry_type'] == 'log':
                        do_insert_log(connection, data)
                cache.commit()
            except Exception:
                cache.rollback()
                raise

        return self._db.runWithConnection(transaction, cache)

    def _get_history_id(self, connection, agent_id, instance_id):
        '''
        Checks own cache for history_id for agent_id and instance_id.
        If information is missing fetch it from database. If it is not there
        create the new record.

        BEWARE: This method runs in a thread.
        '''
        cache_key = (agent_id, instance_id, )
        if cache_key in self._history_id_cache:
            history_id = self._history_id_cache[cache_key]
            return history_id
        else:
            command = text_helper.format_block("""
            SELECT id FROM histories WHERE agent_id = ? AND instance_id = ?
            """)
            cursor = connection.cursor()
            cursor.execute(command, (agent_id, instance_id, ))
            res = cursor.fetchall()
            if res:
                history_id = res[0][0]
                self._history_id_cache[cache_key] = history_id
                return history_id
            else:
                command = 'INSERT INTO histories VALUES (NULL, ?, ?)'
                cursor.execute(command, (agent_id, instance_id, ))
                history_id = cursor.lastrowid
                self._history_id_cache[cache_key] = history_id
                return history_id

    @in_state(State.connected)
    def _flush_next(self):
        if len(self._cache) == 0:
            return defer.succeed(None)
        else:
            d = self._semaphore.run(self._perform_inserts, self._cache)
            d.addCallback(defer.drop_param, self._flush_next)
            return d

class Record(object):
    implements(IRecord)

    def __init__(self, journaler):
        self._journaler = journaler

    def commit(self, **data):
        data['entry_type'] = 'journal'
        self._journaler.insert_entry(**data)

class JournalerConnection(log.Logger, log.LogProxy):
    implements(IJournalerConnection)

    def __init__(self, journaler, externalizer):
        log.LogProxy.__init__(self, journaler)
        log.Logger.__init__(self, self)

        self.serializer = banana.Serializer(externalizer=externalizer)
        self.snapshot_serializer = banana.Serializer()
        self.journaler = IJournaler(journaler)

    ### IJournalerConnection ###

    def new_entry(self, agent_id, instance_id, journal_id, function_id,
                  *args, **kwargs):
        record = self.journaler.prepare_record()
        entry = AgencyJournalEntry(
            self.serializer, record, agent_id, instance_id,
            journal_id, function_id, *args, **kwargs)
        return entry

    def get_filename(self):
        return self.journaler.get_filename()

    def snapshot(self, agent_id, instance_id, snapshot):
        record = self.journaler.prepare_record()
        entry = AgencyJournalEntry(
            self.snapshot_serializer, record, agent_id, instance_id,
            'agency', 'snapshot', snapshot)
        entry.set_result(None)
        entry.commit()

class AgencyJournalSideEffect(object):

    implements(IJournalSideEffect)

    ### IJournalSideEffect ###

    def __init__(self, serializer, record, function_id, *args, **kwargs):
        self._serializer = serializer
        self._record = record
        self._fun_id = function_id
        self._args = serializer.freeze(args or tuple())
        self._kwargs = serializer.freeze(kwargs or dict())
        self._effects = []
        self._result = None

    ### IJournalSideEffect Methods ###

    def add_effect(self, effect_id, *args, **kwargs):
        assert self._record is not None
        data = (effect_id,
                self._serializer.convert(args),
                self._serializer.convert(kwargs))
        self._effects.append(data)

    def set_result(self, result):
        assert self._record is not None
        self._result = self._serializer.convert(result)
        return self

    def commit(self):
        assert self._record is not None
        data = (self._fun_id, self._args, self._kwargs,
                self._effects, self._result)
        self._record.extend(data)
        self._record = None
        return self

class History(formatable.Formatable, pb.Copyable):
    '''
    Mapping for objects in history database.
    '''

    formatable.field('history_id', None)
    formatable.field('agent_id', None)
    formatable.field('instance_id', None)

    @classmethod
    def fetch(cls, db):
        d = db.runQuery(
            "SELECT id, agent_id, instance_id FROM histories")
        d.addCallback(cls._parse_resp)
        return d

    @classmethod
    def _parse_resp(cls, resp):
        columns = map(operator.attrgetter('name'), cls._fields)
        return map(lambda row: cls(**dict(zip(columns, row))), resp)

class AgencyJournalEntry(object):

    implements(IJournalEntry)

    def __init__(self, serializer, record, agent_id, instance_id, journal_id,
                 function_id, *args, **kwargs):
        self._serializer = serializer
        self._record = record

        self._data = {
            'agent_id': agent_id,
            'instance_id': instance_id,
            'journal_id': self._serializer.convert(journal_id),
            'function_id': function_id,
            'fiber_id': None,
            'fiber_depth': None,
            'side_effects': list()}

        self._not_serialized = {
            'args': args or None,
            'kwargs': kwargs or None,
            'result': None}

    ### IJournalEntry Methods ###

    def set_fiber_context(self, fiber_id, fiber_depth):
        assert self._record is not None
        self._data['fiber_id'] = fiber_id
        self._data['fiber_depth'] = fiber_depth
        return self

    def set_result(self, result):
        assert self._record is not None
        self._not_serialized['result'] = result

    def get_result(self):
        return self._not_serialized['result']

    def new_side_effect(self, function_id, *args, **kwargs):
        assert self._record is not None
        record = []
        self._data['side_effects'].append(record)
        return AgencyJournalSideEffect(self._serializer, record,
                                       function_id, *args, **kwargs)

    def commit(self):
        try:
            self._data['args'] = self._serializer.convert(
                    self._not_serialized['args'])
            self._data['kwargs'] = self._serializer.convert(
                    self._not_serialized['kwargs'])
            self._data['result'] = self._serializer.freeze(
                    self._not_serialized['result'])
            self._data['side_effects'] = self._serializer.convert(
                    self._data['side_effects'])
            self._record.commit(**self._data)
            self._record = None
            return self
        except TypeError as e:
            self.set_result(fiber.fail(e))
            self._not_serialized['args'] = None
            self._not_serialized['kwargs'] = None
            self.commit()


import requests,threading,pymongo,random
from lxml.html import etree
from fake_useragent import UserAgent
from concurrent.futures import ThreadPoolExecutor

#连接数据库
client = pymongo.MongoClient('localhost',27017)

#选择book数据库中的urls集合,取出出版社对应的url
read_db = client.book.urls

#选择book_info数据库中的books集合进行保存
save_db = client.book_info.books

#设置线程池
pool = ThreadPoolExecutor(4)

#构造请求头
headers = {
    'User-Agent':UserAgent().chrome
}

#设置线程锁,为了使读取到的数据更精确
lock = threading.RLock()

#定义一个类,实现书籍信息的获取
class GetBookInfo(object):
    '''
    book_urls:临时存放待爬取的url
    pool:代理池
    number:动态设置一次从数据库取出多数数据
    get_data:主执行函数,获取所有书籍数据
    '''
    def __init__(self,number = 50,ip_pool = None):
        self.book_urls = []
        self.start_num = save_db.count()
        self.number = number
        self.pool = ip_pool

    def get_data(self,):
        # 返回True说明还有有未被爬取url
        if self.read_urls():
            # 直到解析完所有,否则一直循环
            while self.book_urls:
                url = self.book_urls.pop()['url']
                print(url)
                # 获取信息并保存
                self.get_url(url)

            #解析完毕后重新执行get_data,直到self.find_urls()返回False说明全部爬取完毕
            return self.get_data()
        else:
            return False

    def get_url(self,url):
        # 判断是否使用代理进行请求

        if self.pool == None:
            response = requests.get(url, headers=headers)
        else:
            response = requests.get(url, headers=headers, proxies=random.choice(self.pool))

        # 使用xpath 解析
        e = etree.HTML(response.text)

        # 获取相关的信息
        book_isbn = e.xpath('//div[@class="min_h300 bookinfo"]/p[4]/text()')[0].split('：')[1]
        book_cip = e.xpath('//div[@class="min_h300 bookinfo"]/p[3]/text()')[0].split('：')[1]
        book_name = e.xpath('//div[@class="min_h300 bookinfo"]/h3/text()')[0]
        book_author = e.xpath('//div[@class="min_h300 bookinfo"]/p[1]/text()')[0].split('：')[1]
        book_publishers = e.xpath('//div[@class="min_h300 bookinfo"]/p/a/text()')[0]
        book_pub_time = e.xpath('//div[@class="min_h300 bookinfo"]/p[6]/text()')[0].split('：')[1]
        book_price = e.xpath('//div[@class="min_h300 bookinfo"]//strong/text()')[0]
        book_synopsis = e.xpath('//div[@id="tab1"]/p/text()')[0]
        book_info = {
            '_id':book_isbn,#获取IBSN编码,由于其唯一性,故可以设置为_id
            'book_cip': book_cip,#获取书籍的cip号
            'book_name': book_name,#获取书名
            'book_author': book_author,#获取作者
            'book_publishers': book_publishers,#获取出版社
            'book_pub_time': book_pub_time,#获取出版时间
            'book_price': book_price,#获取价格
            'book_synopsis': book_synopsis,#获取简介
        }

        #保存至数据库
        self.save_mongodb(book_info)

    def save_mongodb(self,info):
        #利用save的特性去重保存
        save_db.save(info)

    def get_all_data(self):
        # 根据电脑核数开启线程
        f1 = pool.submit(self.get_data)
        f2 = pool.submit(self.get_data)
        f3 = pool.submit(self.get_data)
        f4 = pool.submit(self.get_data)

        # 当所有线程结束时关闭线程池,返回更新的数据数量
        while True:
            if f1.done() and f2.done() and f3.done() and f4.done():
                pool.shutdown()

                # 获取结束时的数据量
                end_num = save_db.count()
                return end_num - self.start_num

    def read_urls(self):
        #加锁,为了防止多线程造成数据混乱
        lock.acquire()
        #获取50条信息并且返回True,如果没有获取到返回Fals
        if list(read_db.find().skip(self.number - 50).limit(50)):
            self.book_urls = list(read_db.find().skip(self.number - 50).limit(50))
            self.number += len(self.book_urls)
            #解锁,并返回True
            lock.release()
            return True
        else:
            # 解锁,并返回False
            lock.release()
            return False

if __name__ == '__main__':
    '''
    测试
    '''
    get_book_info = GetBookInfo()
    data = get_book_info.get_all_data()
    print(get_book_info.number)


import requests
import json
import re
import time

url = 'https://m.weibo.cn/api/container/getIndex?type=uid&value=3495093613&containerid=1076033495093613&page='
headers = {
        'Referer': 'http://m.weibo.cn/u/3495093613',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:44.0) Gecko/20100101 Firefox/44.0'
}

def download(img_url):
    try:
        img_html = requests.get(img_url, headers=headers)
        print(img_url)
    except:
        print("请求图片出错,等待60秒。。。。", img_html.status_code)
        time.sleep(61)
        return
    file_name = img_url.split('/')[-1]
    if img_html.status_code == 200:
        with open('D:/weibo_/%s' % file_name, 'wb') as file:
            file.write(img_html.content)
            print('OK!!!', file_name)
    else:
        print(img_html.status_code, " 等待60？？？")
        time.sleep(60)
        return

for i in range(1, 10000):
    new_url = url + str(i)
    print(new_url, "   ", i)
    res = requests.get(new_url, headers=headers)
    if res.status_code == 200:
        res_json = json.loads(res.text)
        print(res_json)
        reg = re.compile("'url': '(http://w[\w]+\.sinaimg\.cn/large/[\w]+\.[jpgif]{3})")
        img_urls = re.findall(reg, str(res_json))
        if img_urls != []:
            for img_url in img_urls:
                download(img_url)
        else:
            time.sleep(60)
            print("等待60。。。", res.status_code)
            i -= 1
            continue
    else:
        print("等待60！！！")
        time.sleep(60)
        continue


'''
Created on Mar 4, 2015

@author: paul_fang01
'''
from django.conf.urls import patterns, url
from wechat import views

urlpatterns = patterns('',
    url(r'^$', views.welcome),
    url(r'^wechat$', views.wechatDispatcher),
    url(r'^map$', views.welcome),
)


# -*- coding: iso-8859-15 -*-
'''
VISUM add-in Import Google Transit Feed

Date: 22.05.2014
Author: Dimitri Krutik
Contact: Dimitri.Krutik@ptvgroup.com
Company: PTV AG'''

from GoogleTransitImport import GoogleTransitExceptions as ex

class StopsData(object):
    """
    The class represents the container for stop data.
    It managed the stops and stop hierarchy (stop and parent stations).
    """

    def __init__(self):
        # only stops has a stop Area.
        # Stations contains list of the stops -> with stop Areas
        self.__stopToStopArea = {}

        self.__stopToParentstation = {} # stop id: parent station id
        self.__parentstationStops = {} # parent station id : list of the stops

    def addStation(self, stationId):
        stops = self.__parentstationStops.get(stationId)

        if not stops:
            stops = []
            self.__parentstationStops[stationId] = stops

        return stops

    def addStop(self, stopData, stopAreaNo):
        if stopData.parentStationId:
            self.__addStopToStation(stopData)

        self.__stopToStopArea[stopData.stopId] = stopAreaNo

    def __addStopToStation(self, stopData):
        stops = self.addStation(stopData.parentStationId)
        stops.append(stopData.stopId)
        self.__stopToParentstation[stopData.stopId] = stopData.parentStationId

    def isTheSameStation(self, stopId_1, stopId_2):
        """
        Check, if the given stop id's belong to the same station complex.
        """
        parent_1 = self.__findParentStationId(stopId_1)
        parent_2 = self.__findParentStationId(stopId_2)
        return parent_1 != None and parent_2 != None and parent_1 == parent_2

    def __findParentStationId(self, stopId):
        """
        The function searches for the id of the parent station of the given stop id.
        Note, that the given stop id can reference a station itself.
        Return: parent station id, or None, if the stop does not have a parent station.
        """
        # get the parent station of the stop
        parStationId = self.__stopToParentstation.get(stopId)
        if parStationId:
            return parStationId

        # is the stop a parent station
        if self.__parentstationStops.has_key(stopId):
            return stopId

        return None

    def getStopAreas(self, stopId):
        """
        Collect the stop area id's of the given stop.
        The stop id can reference a single stop (one stop area), or
        a station with multiple stops (and multiple stop areas).
        Return: list with stop area id's.
        """
        stopAreaNo = self.__stopToStopArea.get(stopId)
        if stopAreaNo:
            return [stopAreaNo]

        stopAreasNo = self.__findStationStopAreas(stopId)
        if stopAreasNo:
            return stopAreasNo

        mess = _("Referenced stop with id '%s' not found.").decode("iso-8859-15") % stopId
        raise ex.ReferenceNotFound(mess)

    def __findStationStopAreas(self, stationId):
        """
        Search stopArea id's of the stop inside the station.
        """
        stops = self.__parentstationStops.get(stationId)
        if not stops:
            return None

        stopAreasIds = []
        for stopId in stops:
            # Does not need to check if the stop area exists.
            # The mapping "Station -> Stop Id -> stop Area" is created at once.
            stopAreasIds.append(self.__stopToStopArea.get(stopId))

        return stopAreasIds


'''using sklearn
'''
from sklearn import preprocessing

# enc = preprocessing.OrdinalEncoder() # 整数编码
# enc = preprocessing.OneHotEncoder() # 独热编码

genders = ['female', 'male']
locations = ['from Africa', 'from Asia', 'from Europe', 'from US']
browsers = ['uses Chrome', 'uses Firefox', 'uses IE', 'uses Safari']

enc = preprocessing.OneHotEncoder(categories=[genders, locations, browsers])

integer_codes = enc.fit_transform([['female', 'from US', 'uses Safari'],
               ['male', 'from Europe', 'uses Safari']]).toarray()
print(integer_codes)
# [[1. 0. 0. 0. 0. 1. 0. 0. 0. 1.]
# [0. 1. 0. 0. 1. 0. 0. 0. 0. 1.]]

original_representation = enc.inverse_transform([[1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0]])
print(original_representation)
# [['female' 'from US' 'uses Safari']]


# Copyright (c) 2017, Mayo Clinic
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# Redistributions of source code must retain the above copyright notice, this
#     list of conditions and the following disclaimer.
#
#     Redistributions in binary form must reproduce the above copyright notice,
#     this list of conditions and the following disclaimer in the documentation
#     and/or other materials provided with the distribution.
#
#     Neither the name of the Mayo Clinic nor the names of its contributors
#     may be used to endorse or promote products derived from this software
#     without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
# OF THE POSSIBILITY OF SUCH DAMAGE.

import unittest

from jsonasobj import loads
from rdflib import URIRef, Literal, XSD

from fhirtordf.rdfsupport.namespaces import FHIR
from tests.utils.base_test_case import FHIRGraph

json_data = """{
  "resourceType": "VisionPrescription",
  "id": "33123",
  "text": {
    "status": "generated",
    "div": "(cut)"
  },
   "dispense": [
    {
      "product": {
        "coding": [
          {
            "system": "http://hl7.org/fhir/ex-visionprescriptionproduct",
            "code": "lens"
          }
        ]
      },
      "eye": "right",
      "sphere": -2.00,
      "prism": 0.5,
      "base": "down",
      "add": 2.00
    }
  ]
}
  """

class Issue4TestCase(unittest.TestCase):
    # This hasn't been fixed -- failure is expected
    @unittest.expectedFailure
    def test_decimal(self):
        test_json = loads(json_data)
        from fhirtordf.loaders.fhirresourceloader import FHIRResource
        test_rdf = FHIRResource(FHIRGraph(), None, "http://hl7.org/fhir", test_json)
        g = test_rdf.graph
        # rdflib supports decimal precision if you create the data as a string.
        self.assertNotEqual(Literal("2.00", datatype=XSD.decimal), Literal("2.0", datatype=XSD.decimal))
        self.assertEqual(Literal(2.00), Literal(2.0))

        # FHIR requires that *all* decimals use the first form.  While the diopter example above
        # would work, issues arise in situations where numbers really ARE decimal...
        self.assertEqual(Literal("2.00", datatype=XSD.decimal),
                         g.value(
                             g.value(
                                 g.value(URIRef("http://hl7.org/fhir/VisionPrescription/33123"),
                                         FHIR.VisionPrescription.dispense),
                                 FHIR.VisionPrescription.dispense.add), FHIR.value))

if __name__ == '__main__':
    unittest.main()


# -*- coding: utf-8 -*-

import sqlite3
import os
import re
import glob
import yaml
import datetime

from create_db import check_db
from pprint import pprint

db_filename = 'dhcp_snooping.db'
schema_filename = 'dhcp_snooping_schema.sql'
yaml_file = 'switches.yml'
sh_dhcp_snooping = glob.glob('sw*_*')
sh_dhcp_snooping_new = glob.glob('new_data/sw*_*')

def create_list(files_in_dhcp_snooping):
    regex = re.compile('(\S+) +(\S+) +\d+ +\S+ +(\d+) +(\S+)')
    result = []
    for file_name in files_in_dhcp_snooping:
        hostname = re.search('(\S+/)*(\S+)_dhcp_snooping.txt', file_name).group(2)
        with open(file_name) as data:
            for line in data:
                match = regex.search(line)
                if match:
                    result.append(match.groups()+(hostname,)+(1,)+(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),))
    return result

def insert_data_to_switches(path_filename_yml,db):
    with open(path_filename_yml) as f:
        templates = yaml.load(f)
        for row in templates.values():
            result_switches = [(key,value) for key,value in row.items()]
    print('Inserting data to switches_table')
    con = sqlite3.connect(db)
    for row in result_switches:
            with con:
                query = '''INSERT OR REPLACE INTO switches (hostname, location)
                       values (?, ?)'''
                con.execute(query,row)

def insert_data_to_dhcp(result,db):
    print('Inserting data to dhcp_table')
    con = sqlite3.connect(db)
    for row in result:
            with con:
                query = '''INSERT OR REPLACE INTO dhcp (mac, ip, vlan, interface, switch, active, last_active)
                       values (?, ?, ?, ?, ?, ?, ?)'''
                con.execute(query, row)

def update_data_in_dhcp(result,get_mac_spisok,db):
    print('updating data to dhcp_table')
    spisok_updated_mac = [mac[0] for mac in result]
    con = sqlite3.connect(db)
    for old_mac in get_mac_spisok:
        for row in result:
            if old_mac in spisok_updated_mac:
                with con:
                    query = '''update dhcp set ip=:ip, vlan=:vlan, interface=:interface WHERE mac=:mac'''
                    con.execute(query,{'ip':row[1],'vlan':row[2], 'interface':row[3],'mac':row[0]})
            else:
                with con:
                    query1 = '''update dhcp set active=:active WHERE mac=:mac'''
                    con.execute(query1,{'active':0, 'mac':old_mac})

def get_mac_in_dhcp(db):
        spisok = []
        con = sqlite3.connect(db)
        for mac in con.execute('select mac from dhcp'):
            spisok.append(mac[0])
        return spisok

def add_new_column(db):
    print('Adding new column')
    con = sqlite3.connect(db)
    con.execute('ALTER table dhcp ADD COLUMN last_active text')
    con.commit()

spisok_dhcp_snooping = create_list(sh_dhcp_snooping)
check_db(db_filename,schema_filename)
add_new_column(db_filename)
insert_data_to_switches(yaml_file,db_filename)
insert_data_to_dhcp(spisok_dhcp_snooping,db_filename)
get_mac_spisok = get_mac_in_dhcp(db_filename)
new_spisok = create_list(sh_dhcp_snooping_new)
update_data_in_dhcp(new_spisok,get_mac_spisok,db_filename)


'''
PSPNetLite - test
'''

from __future__ import print_function, division

import sys
import subprocess
import pkg_resources

required = {'opencv-python'}
installed = {pkg.key for pkg in pkg_resources.working_set}
missing = required - installed

if missing:
    subprocess.check_call([sys.executable, '-m', 'pip', 'install', *missing], stdout=None)

import matplotlib.pyplot as plt
from patchify import patchify, unpatchify
from pathlib import Path
import pathlib
from PIL import Image
import pandas as pd
import numpy as np
import os
import matplotlib
import pandas as pd
from skimage import io, transform
import numpy as np
import matplotlib.pyplot as plt
import matplotlib as mpl
import rasterio as rio
from rasterio import features
from pathlib import Path
import pathlib
import geopandas as gpd
from descartes import PolygonPatch
from PIL import Image
import itertools
import re
from tqdm.notebook import tqdm
import time
import rasterio as rio

# Albumentations

import albumentations as A
from albumentations.pytorch import ToTensorV2

from tqdm import tqdm
from model import UNET, UNET_manual

from data_loader import Dataloader_trdp, Dataloader_trdp_pre_trained
from losses import CombinedLoss_TRDP

from utils import (
    load_checkpoint,
    load_checkpoint_pspnetlite,
    save_checkpoint,
    check_accuracy,
    save_predictions_as_imgs,
    predict,
    store_predictions,
    store_predictions_with_patching,
    store_predictions_unet_improved,
)

# Pytorch libraries
import torch
import torch.nn as nn  # All neural network modules, nn.Linear, nn.Conv2d, BatchNorm, Loss functions
import torch.optim as optim  # For all Optimization algorithms, SGD, Adam, etc.
import torch.nn.functional as F  # All functions that don't have any parameters
from torch.utils.data import (
    DataLoader,
)  # Gives easier dataset managment and creates mini batches
import torchvision
import torchvision.datasets as datasets  # Has standard datasets we can import in a nice way
from torch.utils.data import Dataset, DataLoader, Sampler # custom dataset handling
import torch.autograd.profiler as profiler # to track model inference and detect leaks
import torchvision.transforms as transforms  # Transformations we can perform on our dataset
from torchvision import datasets, transforms, models
import torchvision.transforms as T
from torchvision import transforms, utils
from torch.autograd import Variable
from torch.nn.modules.padding import ReplicationPad2d
import torchvision.models as models
from torch import optim
from collections import OrderedDict
import segmentation_models_pytorch as smp #semantic segmentation models and utils
from torch.cuda.amp import GradScaler
from torch.cuda.amp import autocast
import torchvision.transforms.functional as TF

print('PyTorch version:', torch.__version__)

# Clean graphics memory
import gc
gc.collect()
torch.cuda.empty_cache()

#import imgaug
import random
# Ignore warnings
import warnings
warnings.filterwarnings("ignore")

plt.ion()   # interactive mode

import os
os.environ["CUDA_VISIBLE_DEVICES"]="0"

import numpy as np
import segmentation_models_pytorch as smp
import random

# Set seeds
def set_seed(seed=0):
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = False
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    np.random.seed(seed)
    random.seed(seed)
'''
def force_cudnn_initialization():
    s = 32
    dev = torch.device('cuda')
    torch.nn.functional.conv2d(torch.zeros(s, s, s, s, device=dev), torch.zeros(s, s, s, s, device=dev))

force_cudnn_initialization()
'''

###############################################################################
##########################Hyperparameters######################################
###############################################################################

in_channel = 3
num_classes = 1
learning_rate = 1e-5
batch_size = 1
num_epochs = 1
chip_dimension = 512
LOAD_MODEL = False
TRAIN = False
TEST = False
SAVE = True
# define threshold to filter weak predictions
THRESHOLD = 0.5

###############################################################################
#########################Set paths to dataset##################################
###############################################################################
'''
# Draft-dataset paths
train_dir  = Path('../../DATASET/dataset_pruebas/train')
test_dir  = Path('../../DATASET/dataset_pruebas/validation')
csv_file = Path('./registros/output_csvs_dataset_prueba/df_train_untidy.csv')
csv_file_test = Path('./registros/output_csvs_dataset_prueba/df_test_untidy.csv')

# Full dataset paths
#test_dir  = Path('../../DATASET/archive/val_final')
#train_dir = Path('../../DATASET/archive/train_final')
#csv_file = Path('./output_csvs_all/df_train_untidy.csv')
#csv_file_test = Path('./output_csvs_all/df_test_untidy.csv')

'''
# Draft-dataset paths
train_dir  = Path('../../DATASET/dataset_pruebas/train')
test_dir  = Path('../../DATASET/dataset_pruebas/validation')
csv_file = Path('./registros/output_csvs_dataset_prueba/df_train_untidy.csv')
csv_file_test = Path('./registros/output_csvs_dataset_prueba/df_test_untidy.csv')

t_dir  = train_dir

df = pd.read_csv(csv_file)
df_test = pd.read_csv(csv_file_test)

sample_dir = Path('../../DATASET/SN7_buildings_train_sample')

# Reconstructiontest_loader
img_size = 1024
# Chip size given batch_size
chip_dim = ((img_size -1)//batch_size + 1)*2
# number of Columns per chip
columns = img_size / chip_dim
# Needed patches to reconstruct original image
patches_total = int(columns**2)

mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
A.Normalize(mean=mean,std=std)
#A.Rotate(limit=(-360, 360), interpolation=4, border_mode=4,p=1),

transform = A.Compose(
    [
        A.PadIfNeeded(min_height=chip_dimension,min_width=chip_dimension,value=0,p=1),
        ToTensorV2()
    ]
)

train_set   = Dataloader_trdp(root_dir=train_dir,csv_file=csv_file,chip_dimension=chip_dimension,transform=transform)
testing_set = Dataloader_trdp(root_dir=test_dir,csv_file=csv_file_test,chip_dimension=chip_dimension,transform=transform)
train_loader = DataLoader(dataset = train_set, batch_size=batch_size, shuffle = False)
test_loader  = DataLoader(dataset = testing_set, batch_size=batch_size, shuffle = False)

print(f"Train : {len(train_loader)} - Test: {len(test_loader)}")

###############################################################################
###############################Training########################################
###############################################################################

class configuration:
    def __init__(self):
        self.experiment_name = "trdp1.001"
        self.pre_load = "True" ## Load dataset in memory
        self.pre_trained = "True"
        self.num_classes = num_classes
        self.ignore_label = 255
        self.lr = learning_rate  # 0.001 if pretrained. 0.1 if scratch
        self.M = [] ##If training from scratch, reduce learning rate at some point
        self.batch_size = batch_size  # Training batch size
        self.test_batch_size = 4  # Test batch size
        self.epoch = num_epochs ## Number of epochs
        self.train_root = "./VOC"
        self.download = False
        self.seed = 271828

## Create arguments object
args = configuration()

# Make sure to enable GPU acceleration!
device = "cuda" if torch.cuda.is_available() else "cpu"

# Set random seed for reproducability
torch.backends.cudnn.deterministic = True  # fix the GPU to deterministic mode
torch.manual_seed(args.seed)  # CPU seed
torch.cuda.manual_seed_all(args.seed)  # GPU seed
random.seed(args.seed)  # python seed for image transformation
np.random.seed(args.seed)

class PSPNetLite(nn.Module):
    def __init__(self, args, num_classes, pretrained=True, use_aux=True):
        super(PSPNetLite, self).__init__()
        self.use_aux = use_aux

        #### TO FILL: define pytorch default resnet-18 architecture (pretrained and not)
        if pretrained=="True":
            resnet = models.resnet18(pretrained=True) #
        else:
            resnet = models.resnet18(pretrained=False) #

        self.layer0 = nn.Sequential(resnet.conv1, resnet.bn1, resnet.relu, resnet.maxpool)
        self.layer1, self.layer2, self.layer3, self.layer4 = resnet.layer1, resnet.layer2, resnet.layer3, resnet.layer4

        for n, m in self.layer3.named_modules():
            if 'conv2' in n:
                m.dilation, m.padding, m.stride = (2, 2), (2, 2), (1, 1)
        for n, m in self.layer4.named_modules():
            if 'conv2' in n:
                m.dilation, m.padding, m.stride = (4, 4), (4, 4), (1, 1)

        ##Pooling module: simplification of Pyramid Pooling Module of PSPnet
        self.pm = nn.Sequential(
            nn.AdaptiveAvgPool2d(3),
            nn.Conv2d(512, 256, kernel_size=1, bias=False),
            nn.BatchNorm2d(256, momentum=.95),
            nn.ReLU(inplace=True)
        )

        ## Final classifier to get per-pixel predictions
        self.final = nn.Sequential(
            nn.Conv2d(768, 512, kernel_size=3, padding=1, bias=False),
            nn.BatchNorm2d(512, momentum=.95), #
            nn.ReLU(inplace=True),
            nn.Dropout(0.1),
            nn.Conv2d(512, 1, kernel_size=1)
        )

    #### To fill: write the forward pass function:
    #### layer0 --> layer1 --> layer2 --> layer3 --> layer4--> pm --> final
    def forward(self, x):
        x_size = x.size()

        x = self.layer0(x) #layer0
        x = self.layer1(x) #layer1
        x = self.layer2(x) #layer2
        x = self.layer3(x) #layer3

        x1 = self.layer4(x)
        x2 = self.pm(x1)

        # Concatenate layer4 features with upsampled Pooling Module features
        x = self.final(torch.cat((x1, F.interpolate(x2, x1.size()[2:], mode='bilinear')), dim=1)) #
        ##return prediction after bilinear upsampling to original size
        return F.interpolate(x, x_size[2:], mode='bilinear')

def _fast_hist(label_pred, label_true, num_classes):
    mask = (label_true >= 0) & (label_true < num_classes)
    hist = np.bincount(
        num_classes * label_true[mask].astype(int) +
        label_pred[mask], minlength=num_classes ** 2).reshape(num_classes, num_classes)
    return hist

def train_SemanticSeg(args, model, device, train_loader, optimizer, epoch):
    # switch to train mode
    model.train()

    train_loss = []
    counter = 1

    criterion = nn.BCEWithLogitsLoss()
    gts_all, predictions_all = [], []

    for batch_idx, (data) in enumerate(train_loader):

        images = data["raster_diff"].float()

        mask = data["mask_diff"].float()

        images, mask = images.to(device), mask.to(device).squeeze()

        #Forward pass
        outputs = model(images).squeeze()

        #Aggregated per-pixel loss
        loss = criterion(outputs, mask)
        train_loss.append(loss.item())

        # compute gradient and do SGD step
        optimizer.zero_grad()

        loss.backward()

        optimizer.step()

        if counter % 15 == 0:
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}, Learning rate: {:.6f}'.format(
                epoch, int(counter * len(images)), len(train_loader.dataset),
                100. * counter / len(train_loader), loss.item(),
                optimizer.param_groups[0]['lr']))
        counter = counter + 1

    return sum(train_loss) / len(train_loss)#, mean_iu

def testing(args, model, device, test_loader):

    # switch to train mode
    model.eval()
    loss_per_batch = []
    test_loss = 0

    ##We ignore index 255, i.e. object contours labeled with 255 in the val GT
    criterion = nn.BCEWithLogitsLoss()
    gts_all, predictions_all = [], []
    with torch.no_grad():
        for batch_idx, (data) in enumerate(test_loader):
            print(f" {batch_idx}/{len(test_loader)} ")
            images = data["raster_diff"].float()
            mask = data["mask_diff"].float()#type(torch.LongTensor)
            images, mask = images.to(device), mask.to(device).squeeze()

            #Forward pass
            outputs = model(images).squeeze()
            #outputs = outputs.clone().detach().cpu().numpy()
    #        outputs = outputs.cpu().numpy()
            outputs = ((outputs - outputs.min())/(outputs.max()-outputs.min()))
            #outputs = np.round(outputs)
            outputs[outputs>THRESHOLD] = 1
            outputs[outputs<=THRESHOLD] = 0
            outputs = torch.round(outputs)

            #Aggregated per-pixel loss
            loss = criterion(outputs, mask)
            loss_per_batch.append(loss.item())

            # Convert to numpy
            outputs = outputs.detach().cpu().numpy()
            outputs = outputs.astype("int64")
            #predictions = np.round(predictions)

            mask = mask.data.cpu().numpy()
            mask = mask.astype("int64")

            '''
            import matplotlib.pyplot as plt
            plt.imshow(outputs)
            np.unique(outputs)

            preds = outputs.data.max(1)[1].squeeze(1).squeeze(0).cpu().numpy()

            mask = mask.data.cpu().numpy()
            mask = mask.astype("int64")
            plt.imshow(mask)

            assert np.unique(mask) == np.unique(outputs*255)
            '''
            gts_all.append(mask)
            predictions_all.append(255*outputs)

            if SAVE:

                images = "outputs"
                labels = images+"/predictions"
                true = images+"/labels"

                if not os.path.exists(images):
                    os.makedirs(images)
                if not os.path.exists(labels):
                        os.makedirs(labels)
                if not os.path.exists(true):
                        os.makedirs(true)

                matplotlib.image.imsave(f"{labels}/Pred_{batch_idx}.png", outputs , cmap='gray')

                matplotlib.image.imsave(f"{true}/True_{batch_idx}.png", mask, cmap='gray')

    #test_loss /= len(test_loader.dataset)
    loss_per_epoch = [np.average(loss_per_batch)]

    iou_scores = []
    for lp, lt in zip(predictions_all, gts_all):
        # Convert lp to cpu
        #lp = lp.detach().cpu().numpy()
        intersection = np.logical_and(lp.flatten(), lt.flatten())
        union = np.logical_or(lp.flatten(), lt.flatten())
        iou_score = np.sum(intersection) / np.sum(union)
        iou_scores.append(iou_score)

    mean_iu = np.average(iou_scores)

    print('\nTest set ({:.0f}): Average loss: {:.4f}, mIoU: {:.4f}\n'.format(
        len(test_loader.dataset), loss_per_epoch[-1], mean_iu))

    ###########################################################################
    # Store predictions

    return loss_per_epoch, mean_iu, predictions_all, gts_all

model = PSPNetLite(1, num_classes=1, pretrained=args.pre_trained).to(device)

print('Total params: %2.fM' % (sum(p.numel() for p in model.parameters()) / 1000000.0))

milestones = args.M
#optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=0.9, weight_decay=1e-4)
optimizer = optim.Adam(model.parameters(), lr=args.lr)

scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=milestones, gamma=0.1)

import os
import time
loss_train_epoch = []
loss_test_epoch = []
acc_train_per_epoch = []
acc_test_per_epoch = []
new_labels = []

cont = 0

res_path = "./metrics_" + args.experiment_name

filename = "psplite_9_dic_lr_-4.pth" # my_checkpoint.pth.tar
model_path = "./models" # + filename

if not os.path.exists(model_path):
    os.makedirs(model_path)
if LOAD_MODEL:
        load_checkpoint_pspnetlite(torch.load("models/" + filename), model)

if not os.path.isdir(res_path):
    os.makedirs(res_path)

for epoch in range(1, args.epoch + 1):
        st = time.time()
        scheduler.step()
        # train for one epoch
        if TRAIN:

            print("Unet improved --> training, epoch " + str(epoch))

            loss_per_epoch = train_SemanticSeg(args, model, device, train_loader, optimizer, epoch)

            loss_train_epoch += [loss_per_epoch]

            if epoch == args.epoch:
                torch.save(model.state_dict(), "models/" + filename)

            np.save(res_path + '/' + 'LOSS_epoch_train.npy', np.asarray(loss_train_epoch))

        # TESTING
        if TEST:
            print("Unet improved ==> testing, epoch " + str(epoch))
            # test
            loss_per_epoch_test, acc_val_per_epoch_i, a,b = testing(args, model, device, test_loader)

            loss_test_epoch += loss_per_epoch_test
            acc_test_per_epoch += [acc_val_per_epoch_i]

            if epoch == 1:
                best_acc_val = acc_val_per_epoch_i

            else:
                if acc_val_per_epoch_i > best_acc_val:
                    best_acc_val = acc_val_per_epoch_i

            np.save(res_path + '/' + 'LOSS_epoch_val.npy', np.asarray(loss_test_epoch))

            # save accuracies:
            np.save(res_path + '/' + 'accuracy_per_epoch_val.npy', np.asarray(acc_test_per_epoch))

            cont += 1

scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=milestones, gamma=0.1)

# PREDICT

y_pred, true_pred = [], []

loss_per_epoch_test, acc_val_per_epoch_i, y_pred, true_pred = testing(args, model, device, test_loader)


class Bubblesort:
    def __init__(self):
        self.compareCount = 0
        self.swap = 0

    def bubble_sort(self, array):
        not_sorted = True
        while not_sorted:
            for i in range(0, len(array) - 2):
                self.compareCount += 1
                if array[i].length < array[i + 1].length:
                    self.swap += 1
                    var = array[i]
                    array[i] = array[i + 1]
                    array[i + 1] = var
            not_sorted = False
        return array


import tensorflow as tf

from experiment.transfer.model import SentenceClassificationModel
from experiment.transfer.model.avg_wordembeddings import non_zero_tokens
from experiment.utils.variables import weight_variable, bias_variable

class CNNMTLModel(SentenceClassificationModel):
    def __init__(self, config, config_global, logger):
        super(CNNMTLModel, self).__init__(config, config_global, logger)
        self.n_filters = self.config['filters']
        self.window_size = self.config['filter_size']

    def build(self, data, sess):
        self.build_input(data, sess)

        self.W_conv1 = weight_variable('W_conv', [self.window_size, self.embedding_size, 1, self.n_filters])
        self.b_conv1 = bias_variable('b_conv', [self.n_filters])

        representation_sentence = tf.reduce_max(
            tf.nn.relu(
                self.cnn_representation_raw(
                    self.embeddings_sentence,
                    self.sentence_length
                )
            ),
            axis=1,
            keep_dims=False
        )

        representation_sentence = tf.nn.dropout(representation_sentence, self.dropout_keep_prob)

        W2_task_A = weight_variable('W2_task_A', [self.n_filters, len(data.classes)])
        b2_task_A = bias_variable('b2_task_A', [len(data.classes)])
        W2_task_B = weight_variable('W2_task_B', [self.n_filters, len(data.classes)])
        b2_task_B = bias_variable('b2_task_B', [len(data.classes)])

        prediction_task_A = tf.nn.xw_plus_b(representation_sentence, W2_task_A, b2_task_A)
        prediction_task_B = tf.nn.xw_plus_b(representation_sentence, W2_task_B, b2_task_B)

        with tf.variable_scope("Loss_A") as scope:
            self.loss_individual_task_A = tf.nn.softmax_cross_entropy_with_logits(
                labels=self.input_label, logits=prediction_task_A
            )
            self.loss_task_A = tf.reduce_mean(self.loss_individual_task_A)

        with tf.variable_scope("Loss_B") as scope:
            self.loss_individual_task_B = tf.nn.softmax_cross_entropy_with_logits(
                labels=self.input_label, logits=prediction_task_B
            )
            self.loss_task_B = tf.reduce_mean(self.loss_individual_task_B)

        self.predict_task_A = tf.nn.softmax(prediction_task_A)
        self.predict_task_B = tf.nn.softmax(prediction_task_B)

        tf.summary.scalar('Loss-Task-A', self.loss_task_A)
        tf.summary.scalar('Loss-Task-B', self.loss_task_B)

    def cnn_representation_raw(self, item, sequence_length):
        """Creates a representation graph which retrieves a text item (represented by its word embeddings) and returns
        a vector-representation

        :param item: the text item
        :param sequence_length: maximum length of the text item
        :return: representation tensor
        """
        # we need to add another dimension, because cnn works on 3d data only
        cnn_input = tf.expand_dims(item, -1)

        convoluted = tf.nn.bias_add(
            tf.nn.conv2d(
                cnn_input,
                self.W_conv1,
                strides=[1, 1, self.embedding_size, 1],
                padding='SAME'
            ),
            self.b_conv1
        )

        return tf.reshape(convoluted, [-1, sequence_length, self.n_filters])

component = CNNMTLModel


#!/usr/bin/env python3

import argparse
from glob import glob
from importlib.util import spec_from_file_location, module_from_spec
from os import path
from re import match
from shutil import copytree
from time import time

DIR = path.dirname(path.abspath(__file__))

def load_all_days():
  days = {}

  for day_path in glob(f'{DIR}/day*/run.py'):
    day = int(match('.+day(\d\d?)', day_path)[1])
    spec = spec_from_file_location(f'day{day}', day_path)
    day_module = module_from_spec(spec)
    spec.loader.exec_module(day_module)

    days[day] = day_module

  return days

def create_from_template(day_num):
  copytree(f'{DIR}/template', f'{DIR}/day{day_num}')

def exec_time(f):
  start = time()
  res = f()
  t = time() - start

  ms = round(t * 1000, 2)

  return (ms, res)

def run_day(days, day_num):
  part1_time, part1_res = exec_time(days[day_num].part1)
  part2_time, part2_res = exec_time(days[day_num].part2)

  print(f'Day {day_num}')
  print(f'  Part 1: {part1_res} - {part1_time} ms')
  print(f'  Part 2: {part2_res} - {part2_time} ms\n')

def run_all_days(days):
  day_keys = list(days.keys())
  day_keys = sorted(day_keys, key=int)

  for day in day_keys:
    run_day(days, day)

def parse_arguments():
  parser = argparse.ArgumentParser()

  parser.add_argument('-d', '--day', help='Run a specific day', type=int)
  parser.add_argument('-r', '--run', help='Alias for --day', type=int, dest='day')
  parser.add_argument('-c', '--create', help='Create a new day folder from the template', type=int)
  parser.add_argument('-a', '--all', help='Run all days', action='store_true')

  return parser.parse_args()

def main():
  days = load_all_days()
  args = parse_arguments()

  if args.create:
    if args.create in days:
      print(f'ERROR! Day "{args.create}" already created!')
      return

    create_from_template(args.create)

  elif args.day:
    run_day(days, args.day)

  elif args.all:
    run_all_days(days)

  else:
    latest_day = max(days.keys(), key=int)
    run_day(days, latest_day)

if __name__ == '__main__':
  main()


from bs4 import BeautifulSoup
import urllib2
from flask import Flask, url_for, render_template
app = Flask(__name__)

@app.route('/')
def index():
    # read the service status doc off of MTA.info
    # doc gets refreshed every min
    r = urllib2.urlopen('http://www.mta.info/status/serviceStatus.txt').read()
    soup = BeautifulSoup(r, "xml")
    status = {}

    # get the name and status as k, v pair of all the available service lines
    for items in soup.find_all('name'):
        status[items.string] = items.find_next_sibling().string

    # return template, with status dict parameter
    return render_template('index.html', status=status)

# remove for heroku
#if __name__ == '__main__':
#    app.run()


import numpy as np

class Sequential:
    def __init__(self):
        self.model = []
        self.y = []
        self.lr = 0.01

    def __call__(self, x_, save=True):
        x = np.zeros((1,len(x_)))
        x[0] = x_
        for layer in self.model:
            #self.y.append(x)
            x = layer(x,save)
        return x

    def add(self, layer):
        self.model.append(layer)

    def compile(self, loss="MSE", optimizer="SGD"):
        self.loss = loss
        self.optimizer = optimizer
        if optimizer == "MomentumSGD":
            self.lr = 0.001
            self.grad = [0. for i in range(len(self.model))]
        elif optimizer == "RMSPropGraves":
            self.eta = 0.00025
            self.alpha = 0.95
            self.beta = 0.95
            self.epsilon = 0.0001
            self.h = [0. for i in range(len(self.model))]
            self.g = [0. for i in range(len(self.model))]
            self.v = [0. for i in range(len(self.model))]
        elif optimizer == "Adam":
            self.lr == 0.001
            self.m = [0. for i in range(len(self.model))]
            self.v = [0. for i in range(len(self.model))]
            self.beta1 = 0.9
            self.beta2 = 0.999
            self.epsilon = 1e-8

    def update(self, loss):
        if self.optimizer == "SGD":
            self.SGD(loss)
        elif self.optimizer == "MomentumSGD":
            self.MomentumSGD(loss)
        elif self.optimizer == "RMSPropGraves":
            self.RMSPropGraves(loss)
        elif self.optimizer == "Adam":
            self.Adam(loss)

    def SGD(self, loss):
        for layer in reversed(self.model):
            grad = np.dot(layer.x.T, loss)
            loss = np.dot(grad, np.ones((grad.shape[1],1))).T
            loss = np.delete(loss,-1,1)
            layer.W -= self.lr * layer.act_d(grad)

    def MomentumSGD(self, loss):
        for i,layer in enumerate(reversed(self.model)):
            grad = np.dot(layer.x.T, loss)
            loss = np.dot(grad, np.ones((grad.shape[1],1))).T
            loss = np.delete(loss,-1,1)
            self.grad[i] = self.lr * grad + 0.9 * self.grad[i]
            layer.W -= self.grad[i]

    def RMSPropGraves(self, loss):
        for i,layer in enumerate(reversed(self.model)):
            grad = np.dot(layer.x.T, loss)
            loss = np.dot(grad, np.ones((grad.shape[1],1))).T
            loss = np.delete(loss,-1,1)
            self.h[i] = self.alpha * self.h[i] + (1.-self.alpha) * (grad**2)
            self.g[i] = self.alpha * self.g[i] + (1.-self.alpha) * grad
            self.v[i] = (-1*self.eta / np.sqrt(self.h[i] - self.g[i]**2 + self.epsilon)) * grad+ self.beta * self.v[i]
            layer.W += self.v[i]

    def Adam(self, loss):
        for i,layer in enumerate(reversed(self.model)):
            grad = np.dot(layer.x.T, loss)
            loss = np.dot(grad, np.ones((grad.shape[1],1))).T
            loss = np.delete(loss,-1,1)
            self.m[i] = self.beta1 * self.m[i] + (1.-self.beta1) * grad
            self.v[i] = self.beta2 * self.v[i] + (1.-self.beta2) * (grad**2)
            m_ = self.m[i]/(1.-self.beta1)
            v_ = self.v[i]/(1.-self.beta2)
            layer.W -= self.lr * m_ / (np.sqrt(v_) + self.epsilon)


import torch
import numpy as np
from utils import *
import torch.nn.functional as F
import time
import csv
import matplotlib.pyplot as plt
from dataloader import mnist as data
from approach.sivi_bnn_mnist import Appr as appr

cnn = False
if cnn:
    from networks.SIVI_bnn_CNN_MNIST import Net
else:
    from networks.SIVI_bnn_MNIST import Net

##################init
"""
better exp:
  relu -> leaky

"""
tstart = time.time()
# Seed
np.random.seed(0)
torch.manual_seed(0)
if torch.cuda.is_available():
    torch.cuda.manual_seed(0)
else:
    print('[CUDA unavailable]'); sys.exit()

##################################################
"""
#train 97.x, valid 96.x, test 96.x
lr = 0.001
nepochs = 150
sbatch = 256
lr_min=2e-6
lr_factor=3
lr_patience=5
nosample = 10
hid_layer = 100
optim = 'Adam'
SIVI_layer_size = 100
SIVI_input_dim = 40
semi_unit = False
prior_gmm = False
semi_by_col = False

#best: 98.5, valid 97.5, test 97.4
lr = 0.001
nepochs = 300
sbatch = 256
lr_min=2e-6
lr_factor=3
lr_patience=5
nosample = 10
hid_layer = 100
optim = 'Adam'
SIVI_layer_size = 400
SIVI_input_dim = 100
semi_unit = False
prior_gmm = False
semi_by_col = False
"""

lr_min = 1e-7
lr_factor=3
lr_patience=5
optim = 'Adam'

semi_unit = False
semi_by_col = False #if true: semi followed by cols else followed by rows

###################load data
print("Load data...")
print('MNIST DATASET')

dat, input_dim, output_dim = data.get() #read data
xtrain = dat['train']['x'].cuda()
ytrain = dat['train']['y'].cuda()

xvalid = dat['valid']['x'].cuda()
yvalid = dat['valid']['y'].cuda()

xtest = dat['test']['x'].cuda()
ytest = dat['test']['y'].cuda()

print("Train: "+ str(xtrain.shape))
print("Valid: "+ str(xvalid.shape))
print("Test: "+ str(xtest.shape))
print('Done!')
print('\n'+ '*'*200)

hid_layer = 400

##########################Tune parameters
nepochs = 600
lr = 0.0001
droprate = None
sbatch = 256
train_sample = 10
test_sample = 20
test_w_sample = 10
SIVI_input_dim = 400
SIVI_layer_size = 400
re_wKL = 1

local_rep = False
prior_gmm = False #prior is gau mixture (True) or gau unit 0,1 (False)

for re_wKL in [0.1, 0.2, "adaptive"]:
#####################init model and apply approach
    torch.set_default_tensor_type('torch.cuda.FloatTensor')
    if cnn:
        model = Net(input_dim, output_dim, prior_gmm=prior_gmm, SIVI_layer_size=SIVI_layer_size, SIVI_input_dim=SIVI_input_dim, re_wKL=re_wKL,droprate= droprate, local_rep= local_rep).cuda()
    else:
        model = Net(input_dim, [hid_layer,hid_layer], output_dim, prior_gmm=prior_gmm,SIVI_by_col=semi_by_col, SIVI_layer_size=SIVI_layer_size, SIVI_input_dim=SIVI_input_dim, re_wKL=re_wKL, semi_unit= semi_unit, droprate= droprate, local_rep= local_rep).cuda()

    Appr = appr(model, optim= optim, train_sample= train_sample, test_sample= test_sample, w_sample= test_w_sample, nepochs= nepochs, sbatch= sbatch, lr=lr, lr_min=lr_min, lr_factor=lr_factor, lr_patience=lr_patience)

    #report model info
    print_model_report(model)
    print_optimizer_config(Appr.optimizer)

    print("Parameters:")
    cnt_param = 0
    for name, param in model.named_parameters():
        if param.requires_grad:
            cnt_param += 1
            print("\t"+ name)
    print('\tTotal: '+ str(cnt_param))
    print('-'*200)
    print("Approach parameters: \n\toptim= {} \n\tlr= {} \n\ttrain_sample= {} \n\ttest_sample= {} \n\ttest_w_sample= {} \n\tnepochs= {} \n\tsbatch= {}".format(optim, lr, train_sample, test_sample, test_w_sample, nepochs, sbatch), end='\n')
    print("Model parameters: \n\thid_layer= {} \n\tSIVI_layer_size= {} \n\tSIVI_input_dim= {} \n\tre_wKL= {} \n\tprior_gmm= {} \n\tlocal_rep= {} \n\tdroprate= {}".format(hid_layer, SIVI_layer_size, SIVI_input_dim, re_wKL, prior_gmm, local_rep, droprate),end='\n')
    print("-"*200)
    ##################### TRAIN
    print('TRAINING')
    Appr.train(xtrain,ytrain,xvalid,yvalid,xtest, ytest)

    print('*'*200)
    ###################### TEST
    print('TESTING')
    train_loss, train_acc = Appr.eval(xtrain, ytrain, test= True)
    valid_loss, valid_acc = Appr.eval(xvalid, yvalid, test= True)
    test_loss, test_acc = Appr.eval(xtest, ytest, test= True)
    print("Test: loss= {:.3f}, acc={:.3f}".format(test_loss,100*test_acc),end= '\n')
    print(Appr.model.fc1.weight_mu, Appr.model.fc1.weight_rho)
    print(Appr.model.fc2.weight_mu, Appr.model.fc2.weight_rho)
    print(Appr.model.fc3.weight_mu, Appr.model.fc3.weight_rho)

    if cnn:
        f = open("result/mnist/mnist_cnn.txt", "a")
        f.write("*Tune parameters: droprate= {}, lr= {}, SIVI_layer_size= {}, SIVI_input_dim= {}, sbatch= {}, train_sample={}, test_sample= {}, test_w_sample= {}, local_rep={}, prior_gmm={}, n_epochs= {}\n result: rain_acc= {}, valid_acc= {},test_acc= {}\n".format(droprate,lr,SIVI_layer_size, SIVI_input_dim, sbatch, train_sample, test_sample, test_w_sample, str(local_rep), str(prior_gmm),nepochs,train_acc,valid_acc,test_acc))
        f.close()
        with open('result/mnist/mnist_cnn.csv', mode='a') as rs_file:
            rs = csv.writer(rs_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
            rs.writerow([str(droprate),str(lr),str(SIVI_layer_size),str(SIVI_input_dim),str(train_sample),str(test_sample),str(test_w_sample),str(local_rep),str(prior_gmm),str(nepochs),str(sbatch),str(train_acc),str(valid_acc),str(test_acc)])
    else:
        f = open("result/mnist/mnist_test.txt", "a")
        f.write("Note: mu0, sig0 -> muy_w (no dnn) and prior 0,1 and re_wKL\n*Tune parameters: re_wKL = {}, droprate= {}, lr= {}, SIVI_layer_size= {}, SIVI_input_dim= {}, sbatch= {}, train_sample={}, test_sample= {}, test_w_sample= {}, local_rep={}, prior_gmm={}, n_epochs= {}\n result: train_acc= {}, valid_acc= {},test_acc= {}\n".format(re_wKL,droprate,lr,SIVI_layer_size, SIVI_input_dim, sbatch, train_sample, test_sample, test_w_sample, str(local_rep), str(prior_gmm),nepochs,train_acc,valid_acc,test_acc))
        f.close()
        # with open('result/mnist/mnist.csv', mode='a') as rs_file:
        #     rs = csv.writer(rs_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
        #     rs.writerow([str(droprate),str(lr),str(SIVI_layer_size),str(SIVI_input_dim),str(train_sample),str(test_sample),str(test_w_sample),str(local_rep),str(prior_gmm),str(nepochs),str(sbatch),str(train_acc),str(valid_acc),str(test_acc)])


import numpy as np
import matplotlib.pyplot as plt
import json
from matplotlib.backends.backend_pdf import PdfPages

def getCandidates(qoeDat):
	candidates = qoeDat[qoeDat.keys()[0]]
	return candidates

def getSrvQoE(qoeDat, candidates):
	chunkID = qoeDat.keys()
	chunkID.sort(key=int)
	srv_qoes = {}
	for c in candidates:
		srv_qoe = [qoeDat[k][c] for k in chunkID]
		srv_qoes[c] = srv_qoe
	return srv_qoes

def getChunkID(qoeDat):
	chunkID = qoeDat.keys()
	chunkID.sort(key=int)
	return chunkID

dataFolder = "./data/test/"

qas_dash_srvqoe = json.load(open(dataFolder + "test1-exp1-QAS_DASH-BBB-srvqoe.json"))
cqas_dash_srvqoe = json.load(open(dataFolder + "test2-exp1-CQAS_DASH-BBB-srvqoe.json"))

chunkid_qas_dash = getChunkID(qas_dash_srvqoe)
candidate_srvs = getCandidates(qoeDat)
qas_srv_qoes = getSrvQoE(qoeDat, candidate_srvs)
chunkid_cqas_dash = getChunkID(cqas_dash_srvqoe)
cqas_srv_qoes = getSrvQoE(cqas_dash_srvqoe, candidate_srvs)

plt.figure(1)
ax = plt.subplot(211)
line1 = plt.plot(chunkid_qas_dash, qas_srv_qoes[candidates[0]], 'b-', label=candidates[0])
line2 = plt.plot(chunkid_qas_dash, qas_srv_qoes[candidates[1]], 'k--', label=candidates[1])
plt.setp(line1, linewidth=2.0)
plt.setp(line2, linewidth=3.0)
plt.legend(bbox_to_anchor=(1, 0.25))
ax.set_xlabel(r'Chunk No.', fontsize=20)
ax.set_ylabel(r'QoE Evaluations on Candidate Servers', fontsize=20)
ax.title(r'QAS-DASH Server Evaluations')

ax = plt.subplot(212)
line1 = plt.plot(chunkid_cqas_dash, cqas_srv_qoes[candidates[0]], 'b-', label=candidates[0])
line2 = plt.plot(chunkid_cqas_dash, cqas_srv_qoes[candidates[1]], 'k--', label=candidates[1])
plt.setp(line1, linewidth=2.0)
plt.setp(line2, linewidth=3.0)
plt.legend(bbox_to_anchor=(1, 0.25))
ax.set_xlabel(r'Chunk No.', fontsize=20)
ax.set_ylabel(r'QoE Evaluations on Candidate Servers', fontsize=20)
ax.title(r'CQAS-DASH Server Evaluations')
plt.show()

pdf = PdfPages('./imgs/srv_qoes_plot.pdf')
pdf.savefig(fig)

pdf.close()


import os,sys,inspect
current_dir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
parent_dir = os.path.dirname(current_dir)
sys.path.insert(0, parent_dir)
import starter

if starter.isInTargetFiles("txt") and not starter.isInTargetFiles("okan"):
    print("Binary Search TEST IS COMPLETED SUCCESSFULLY")
else:
    print("Binary Search TEST IS FAILED!")

if starter.isValidFile("test.txt") and not starter.isValidFile("test.okan"):
    print("Valid File TEST IS COMPLETED SUCCESSFULLY")
else:
    print("Valid File TEST IS FAILED!")


# -*- encoding: utf-8 -*-

from odoo import models, fields, api, _
import logging

_logger = logging.getLogger(__name__)

class FBSettings(models.Model):
	_name = 'prod_fb_post.fb_info'

	page_id = fields.Char(string="Page ID")
	group_id = fields.Char(string="Group ID")
	app_id = fields.Char(string="App ID")
	secret_key = fields.Text(string='Secret Key')
	app_ver = fields.Char(string="App Version")

class ResConfig(models.TransientModel):
	_name = 'prod_fb_post.fb_settings'
	_inherit = "res.config.settings"
	_description = "Facebook connection info"

	fb_info_id = fields.Many2one('prod_fb_post.fb_info', string='FB Info')
	fb_info_page_id = fields.Char(related='fb_info_id.page_id')
	fb_info_group_id = fields.Char(related='fb_info_id.group_id')
	fb_info_app_id = fields.Char(related='fb_info_id.app_id')
	fb_info_secret_key = fields.Text(related='fb_info_id.secret_key')
	fb_info_app_ver = fields.Char(related='fb_info_id.app_ver')

	def create(self, cr, uid, vals, context=None):
		_logger.critical('CREATE' + str (vals))
		config_id = super(ResConfig, self).create(cr, uid, vals, context=context)
		self.write(cr, uid, config_id, vals, context=context)
		return config_id

	_defaults = {
		'fb_info_id': lambda self, cr, uid, c: self.pool.get('prod_fb_post.fb_info').search(cr, uid, [], context=c)[0]
	}


from xai.brain.wordbase.nouns._gadfly import _GADFLY

#calss header
class _GADFLIES(_GADFLY, ):
	def __init__(self,):
		_GADFLY.__init__(self)
		self.name = "GADFLIES"
		self.specie = 'nouns'
		self.basic = "gadfly"
		self.jsondata = {}