"""Object-Rlational Mapping classess, based on Sqlalchemy, for representing the
dataset, partitions, configuration, tables and columns.

Copyright (c) 2013 Clarinova. This file is licensed under the terms of the
Revised BSD License, included in this distribution as LICENSE.txt
"""
__docformat__ = 'restructuredtext en'
import datetime
import sqlalchemy
from sqlalchemy import orm
from sqlalchemy import event
from sqlalchemy import Column as SAColumn, Integer, Float, Boolean, UniqueConstraint
from sqlalchemy import Float as Real, Text, String, ForeignKey, Binary, Enum
from sqlalchemy.orm import relationship
from sqlalchemy.types import TypeDecorator, TEXT, PickleType
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.ext.mutable import Mutable
from sqlalchemy.exc import OperationalError
from util import Constant, memoize
from identity import LocationRef

from sqlalchemy.sql import text
import sqlalchemy.types as types
from ambry.identity import DatasetNumber, ColumnNumber
from ambry.identity import TableNumber, PartitionNumber, ObjectNumber

import json

# http://stackoverflow.com/a/23175518/1144479
# SQLAlchemy does not map BigInt to Int by default on the sqlite dialect.
# It should, but it doesnt.

from sqlalchemy import BigInteger
from sqlalchemy.dialects import postgresql, mysql, sqlite

BigIntegerType = BigInteger()
BigIntegerType = BigIntegerType.with_variant(postgresql.BIGINT(), 'postgresql')
BigIntegerType = BigIntegerType.with_variant(mysql.BIGINT(), 'mysql')
BigIntegerType = BigIntegerType.with_variant(sqlite.INTEGER(), 'sqlite')

from sqlalchemy import func
from sqlalchemy.types import UserDefinedType

class Geometry(UserDefinedType):

    """Geometry type, to ensure that WKT text is properly inserted into the
    database with the GeomFromText() function.

    NOTE! This is paired with code in
    database.relational.RelationalDatabase.table() to convert NUMERIC
    fields that have the name 'geometry' to GEOMETRY types. Sqlalchemy
    sees spatialte GEOMETRY types as NUMERIC

    """

    DEFAULT_SRS = 4326

    def get_col_spec(self):
        return "GEOMETRY"

    def bind_expression(self, bindvalue):
        return func.ST_GeomFromText(bindvalue, self.DEFAULT_SRS, type_=self)

    def column_expression(self, col):
        return func.ST_AsText(col, type_=self)

class SpatialiteGeometry(Geometry):

    def get_col_spec(self):
        return "BLOB"

GeometryType = Geometry()
GeometryType = GeometryType.with_variant(SpatialiteGeometry(), 'spatialite')
GeometryType = GeometryType.with_variant(
    Text(),
    'sqlite')  # Just write the WKT through
GeometryType = GeometryType.with_variant(Text(), 'postgresql')

Base = declarative_base()

class JSONEncoder(json.JSONEncoder):

    """A JSON encoder that turns unknown objets into a string representation of
    the type."""

    def default(self, o):
        from ambry.identity import Identity

        try:
            return o.dict
        except AttributeError:
            return str(type(o))

class JSONEncodedObj(TypeDecorator):

    "Represents an immutable structure as a json-encoded string."

    impl = TEXT

    def process_bind_param(self, value, dialect):
        if value is not None:
            value = json.dumps(value, cls=JSONEncoder)
        else:
            value = '{}'
        return value

    def process_result_value(self, value, dialect):
        if value is not None:
            try:
                value = json.loads(value)
            except:
                # We've changed from using pickle to json, so this handles
                # legacy cases
                import pickle
                value = pickle.loads(value)

        else:
            value = {}
        return value

class MutationDict(Mutable, dict):

    @classmethod
    def coerce(cls, key, value):  # @ReservedAssignment
        """Convert plain dictionaries to MutationDict."""

        if not isinstance(value, MutationDict):
            if isinstance(value, dict):
                return MutationDict(value)

            # this call will raise ValueError
            return Mutable.coerce(key, value)
        else:
            return value

    def __setitem__(self, key, value):
        """Detect dictionary set events and emit change events."""
        dict.__setitem__(self, key, value)

        self.changed()

    def __delitem__(self, key):
        """Detect dictionary del events and emit change events."""

        dict.__delitem__(self, key)
        self.changed()

class MutationObj(Mutable):

    @classmethod
    def coerce(cls, key, value):
        if isinstance(value, dict) and not isinstance(value, MutationDict):
            return MutationDict.coerce(key, value)
        if isinstance(value, list) and not isinstance(value, MutationList):
            return MutationList.coerce(key, value)
        return value

    @classmethod
    def _listen_on_attribute(cls, attribute, coerce, parent_cls):
        key = attribute.key
        if parent_cls is not attribute.class_:
            return

        # rely on "propagate" here
        parent_cls = attribute.class_

        def load(state, *args):
            val = state.dict.get(key, None)
            if coerce:
                val = cls.coerce(key, val)
                state.dict[key] = val
            if isinstance(val, cls):
                val._parents[state.obj()] = key

        def set(target, value, oldvalue, initiator):
            if not isinstance(value, cls):
                value = cls.coerce(key, value)
            if isinstance(value, cls):
                value._parents[target.obj()] = key
            if isinstance(oldvalue, cls):
                oldvalue._parents.pop(target.obj(), None)
            return value

        def pickle(state, state_dict):
            val = state.dict.get(key, None)
            if isinstance(val, cls):
                if 'ext.mutable.values' not in state_dict:
                    state_dict['ext.mutable.values'] = []
                state_dict['ext.mutable.values'].append(val)

        def unpickle(state, state_dict):
            if 'ext.mutable.values' in state_dict:
                for val in state_dict['ext.mutable.values']:
                    val._parents[state.obj()] = key

        sqlalchemy.event.listen(
            parent_cls,
            'load',
            load,
            raw=True,
            propagate=True)
        sqlalchemy.event.listen(
            parent_cls,
            'refresh',
            load,
            raw=True,
            propagate=True)
        sqlalchemy.event.listen(
            attribute,
            'set',
            set,
            raw=True,
            retval=True,
            propagate=True)
        sqlalchemy.event.listen(
            parent_cls,
            'pickle',
            pickle,
            raw=True,
            propagate=True)
        sqlalchemy.event.listen(
            parent_cls,
            'unpickle',
            unpickle,
            raw=True,
            propagate=True)

class MutationList(MutationObj, list):

    @classmethod
    def coerce(cls, key, value):
        """Convert plain list to MutationList."""
        self = MutationList((MutationObj.coerce(key, v) for v in value))
        self._key = key
        return self

    def __setitem__(self, idx, value):
        list.__setitem__(self, idx, MutationObj.coerce(self._key, value))
        self.changed()

    def __setslice__(self, start, stop, values):
        list.__setslice__(
            self,
            start,
            stop,
            (MutationObj.coerce(
                self._key,
                v) for v in values))
        self.changed()

    def __delitem__(self, idx):
        list.__delitem__(self, idx)
        self.changed()

    def __delslice__(self, start, stop):
        list.__delslice__(self, start, stop)
        self.changed()

    def append(self, value):
        list.append(self, MutationObj.coerce(self._key, value))
        self.changed()

    def insert(self, idx, value):
        list.insert(self, idx, MutationObj.coerce(self._key, value))
        self.changed()

    def extend(self, values):
        list.extend(self, (MutationObj.coerce(self._key, v) for v in values))
        self.changed()

    def pop(self, *args, **kw):
        value = list.pop(self, *args, **kw)
        self.changed()
        return value

    def remove(self, value):
        list.remove(self, value)
        self.changed()

def JSONAlchemy(sqltype):
    """A type to encode/decode JSON on the fly.

    sqltype is the string type for the underlying DB column.

    You can use it like:
    Column(JSONAlchemy(Text(600)))

    """
    class _JSONEncodedObj(JSONEncodedObj):
        impl = sqltype

    return MutationObj.as_mutable(_JSONEncodedObj)

class SavableMixin(object):

    def save(self):
        self.session.commit()

class LinkableMixin(object):

    """A mixin for creating acessors to link between objects with references in
    the .dataproperty Should probably be a descriptor, but I don't feel like
    fighting with it."""

    # _get_link_array(self, name, clz, id_column):
    # _append_link(self, name, object_id):
    # _remove_link(self, name, object_id):

    def _get_link_array(self, name, clz, id_column):
        """
        name: the name of the link, a key in the data property
        clz: Sqlalchemy ORM class for the foreign object
        id_column, the Sqlalchemy property, from the clz classs, that hpolds the stored id value for the object.
        """
        from sqlalchemy.orm import object_session

        id_values = self.data.get(name, [])

        if not id_values:
            return []

        return object_session(self).query(clz).filter(id_column.in_(id_values)).all()

    def _append_link(self, name, object_id):
        """
        name: the name of the link, a key in the data property
        o: the object being linked. If none, no back link is made
        object_id: the object identitifer that is stored in the data property
        """
        if not name in self.data:
            self.data[name] = []

        if not object_id in self.data[name]:
            self.data[name] = self.data[name] + [object_id]

    def _remove_link(self, name, object_id):
        """For linking manifests to stores."""
        if not name in self.data:
            return

        if self.data[name] and object_id in self.data[name]:
            self.data[name] = self.data[name].remove(object_id)

class DataPropertyMixin(object):

    """A Mixin for appending a value into a list in the data field."""

    def _append_string_to_list(self, sub_prop, value):
        """"""
        if not sub_prop in self.data:
            self.data[sub_prop] = []

        if value and not value in self.data[sub_prop]:
            self.data[sub_prop] = self.data[sub_prop] + [value]

# Sould have things derived from this, once there are test cases for it.
# Actually, this is a mixin.

class DictableMixin(object):

    def set_attributes(self, **kwargs):
        for k, v in kwargs.items():
            setattr(self, k, v)

    @property
    def dict(self):
        from sqlalchemy.orm.attributes import InstrumentedAttribute
        import inspect

        return dict(
            inspect.getmembers(
                self.__class__,
                lambda x: isinstance(
                    x,
                    InstrumentedAttribute)))

    def __repr__(self):
        return "<{}: {}>".format(type(self), self.dict)

class Dataset(Base, LinkableMixin):
    __tablename__ = 'datasets'

    LOCATION = Constant()
    LOCATION.LIBRARY = LocationRef.LOCATION.LIBRARY
    LOCATION.PARTITION = LocationRef.LOCATION.PARTITION
    LOCATION.SREPO = LocationRef.LOCATION.SREPO
    LOCATION.SOURCE = LocationRef.LOCATION.SOURCE
    LOCATION.REMOTE = LocationRef.LOCATION.REMOTE
    LOCATION.UPSTREAM = LocationRef.LOCATION.UPSTREAM

    vid = SAColumn('d_vid', String(20), primary_key=True)
    id_ = SAColumn('d_id', String(20), )
    name = SAColumn('d_name', String(200), nullable=False, index=True)
    vname = SAColumn('d_vname',String(200),unique=True,nullable=False,index=True)
    fqname = SAColumn('d_fqname', String(200), unique=True, nullable=False)
    cache_key = SAColumn('d_cache_key',String(200),unique=True,nullable=False,index=True)
    source = SAColumn('d_source', String(200), nullable=False)
    dataset = SAColumn('d_dataset', String(200), nullable=False)
    subset = SAColumn('d_subset', String(200))
    variation = SAColumn('d_variation', String(200))
    btime = SAColumn('d_btime', String(200))
    bspace = SAColumn('d_bspace', String(200))
    creator = SAColumn('d_creator', String(200), nullable=False)
    revision = SAColumn('d_revision', Integer, nullable=False)
    version = SAColumn('d_version', String(20), nullable=False)

    data = SAColumn('d_data', MutationDict.as_mutable(JSONEncodedObj))

    path = None  # Set by the LIbrary and other queries.

    tables = relationship("Table",backref='dataset',cascade="delete, delete-orphan")

    partitions = relationship("Partition",backref='dataset',cascade="delete, delete-orphan")

    configs = relationship('Config', backref='dataset', cascade="delete, delete-orphan",
                           primaryjoin="Config.d_vid == Dataset.vid ", foreign_keys="Config.d_vid")

    files = relationship('File', backref='dataset', cascade="delete, delete-orphan",
                         primaryjoin="File.ref == Dataset.vid ", foreign_keys="File.ref")

    #__table_args__ = (
    #    UniqueConstraint('d_vid', 'd_location', name='u_vid_location'),
    #    UniqueConstraint('d_fqname', 'd_location', name='u_fqname_location'),
    #    UniqueConstraint('d_cache_key', 'd_location', name='u_cache_location'),
    #)

    def __init__(self, **kwargs):
        self.id_ = kwargs.get("oid", kwargs.get("id", kwargs.get("id_", None)))
        self.vid = kwargs.get("vid", None)
        # Deprecated?
        self.location = kwargs.get("location", self.LOCATION.LIBRARY)
        self.name = kwargs.get("name", None)
        self.vname = kwargs.get("vname", None)
        self.fqname = kwargs.get("fqname", None)
        self.cache_key = kwargs.get("cache_key", None)
        self.source = kwargs.get("source", None)
        self.dataset = kwargs.get("dataset", None)
        self.subset = kwargs.get("subset", None)
        self.variation = kwargs.get("variation", None)
        self.btime = kwargs.get("btime", None)
        self.bspace = kwargs.get("bspace", None)
        self.creator = kwargs.get("creator", None)
        self.revision = kwargs.get("revision", None)
        self.version = kwargs.get("version", None)

        if self.vid and not self.id_:
            self.revision = ObjectNumber.parse(self.vid).revision
            self.id_ = str(ObjectNumber.parse(self.vid).rev(None))

        if not self.id_:
            dn = DatasetNumber(None, self.revision)
            self.vid = str(dn)
            self.id_ = str(dn.rev(None))
        elif not self.vid:
            try:
                self.vid = str(ObjectNumber.parse(self.id_).rev(self.revision))
            except ValueError as e:
                raise ValueError('Could not parse id value; ' + e.message)

        if self.cache_key is None:
            self.cache_key = self.identity.cache_key

        if not self.name:
            self.name = str(self.identity.name)

        if not self.vname:
            self.vname = str(self.identity.vname)

        if not self.fqname:
            self.fqname = str(self.identity.fqname)

        if not self.version:
            self.version = str(self.identity.version)

        assert self.vid[0] == 'd'

    def __repr__(self):
        return """<datasets: id={} vid={} name={} source={} ds={} ss={} var={} creator={} rev={}>""".format(
            self.id_,
            self.vid,
            self.name,
            self.source,
            self.dataset,
            self.subset,
            self.variation,
            self.creator,
            self.revision)

    @property
    def identity(self):
        from identity import Identity
        return Identity.from_dict(self.dict)

    @property
    def dict(self):
        d = {
            'id': self.id_,
            'vid': self.vid,
            'name': self.name,
            'vname': self.vname,
            'fqname': self.fqname,
            'cache_key': self.cache_key,
            'source': self.source,
            'dataset': self.dataset,
            'subset': self.subset,
            'variation': self.variation,
            'btime': self.btime,
            'bspace': self.bspace,
            'creator': self.creator,
            'revision': self.revision,
            'version': self.version,
        }

        if self.data:
            for k in self.data:
                assert k not in d
                d[k] = self.data[k]

        return d

    def config(self,  key, group = 'config'):
        from sqlalchemy.orm import object_session
        s = object_session(self)
        return (s.query(Config)
              .filter(Config.d_vid == self.vid)
              .filter(Config.group == group )
              .filter(Config.key == key )
              .one())

    # For linking partitions to manifests
    @property
    def linked_manifests(self):
        return self._get_link_array('manifests', File, File.ref)

    def link_manifest(self, f):
        return self._append_link('manifests', f.ref)

    def delink_manifest(self, f):
        return self._remove_link('manifests', f.ref)

    @property
    def linked_stores(self):
        return self._get_link_array('stores', File, File.ref)

    def link_store(self, f):
        return self._append_link('stores', f.ref)

    def delink_store(self, f):
        return self._remove_link('stores', f.ref)

def _clean_flag(in_flag):

    if in_flag is None or in_flag == '0':
        return False

    return bool(in_flag)

class Code(Base, SavableMixin, LinkableMixin):

    """Code entries for variables."""
    __tablename__ = 'codes'

    c_vid = SAColumn('cd_c_vid',String(20),ForeignKey('columns.c_vid'), primary_key=True,index=True,nullable=False)

    d_vid = SAColumn('cd_d_vid', String(20), ForeignKey('datasets.d_vid'), primary_key=True, nullable=False, index=True)

    key = SAColumn('cd_skey',String(20), primary_key=True,nullable=False,index=True)  # String version of the key, the value in the dataset
    ikey = SAColumn( 'cd_ikey',Integer,index=True)  # Set only if the key is actually an integer

    value = SAColumn('cd_value', Text,nullable=False)  # The value the key maps to
    description = SAColumn('f_description', Text, index=True)

    data = SAColumn('co_data', MutationDict.as_mutable(JSONEncodedObj))

    def __init__(self, **kwargs):

        for p in self.__mapper__.attrs:
            if p.key in kwargs:
                setattr(self, p.key, kwargs[p.key])
                del kwargs[p.key]

        if self.data:
            self.data.update(kwargs)

    def __repr__(self):
        return "<code: {}->{} >".format(self.key, self.value)

    def update(self, f):
        """Copy another files properties into this one."""

        for p in self.__mapper__.attrs:

            if p.key == 'oid':
                continue
            try:
                setattr(self, p.key, getattr(f, p.key))

            except AttributeError:
                # The dict() method copies data property values into the main dict,
                # and these don't have associated class properties.
                continue

    @property
    def dict(self):

        d = { p.key: getattr( self,p.key)
             for p in self.__mapper__.attrs if p.key not in ('data','column','table')
        }

        if self.data:
            for k in self.data:
                assert k not in d
                d[k] = self.data[k]

        return d

    @property
    def insertable_dict(self):

        d =  {('cd_' + k).strip('_'): v for k, v in self.dict.items()}

        # the `key` property is not named after its db column
        d['cd_skey'] = d['cd_key']
        del d['cd_key']

        return d

    @staticmethod
    def before_insert(mapper, conn, target):

        target.d_vid = str(ObjectNumber.parse(target.c_vid).as_dataset)

event.listen(Code, 'before_insert', Code.before_insert)

lom_enums = "nom ord int ratio".split()

class ColumnStat(Base, SavableMixin, LinkableMixin):

    """Table for per column, per partition stats."""
    __tablename__ = 'colstats'

    p_vid = SAColumn('cs_p_vid',String(20),ForeignKey('partitions.p_vid'), primary_key=True, nullable=False, index=True)
    partition = relationship('Partition', backref='_stats')

    c_vid = SAColumn('cs_c_vid', String(20), ForeignKey('columns.c_vid'), primary_key=True, nullable=False, index=True)

    d_vid = SAColumn('cs_d_vid', String(20), ForeignKey('datasets.d_vid'), nullable=False, index=True)

    lom = SAColumn('cs_lom', String(12))
    count = SAColumn('cs_count', BigIntegerType)
    mean = SAColumn('cs_mean', Float)
    std = SAColumn('cs_std', Float)
    min = SAColumn('cs_min', BigIntegerType)
    p25 = SAColumn('cs_p25', BigIntegerType)
    p50 = SAColumn('cs_p50', BigIntegerType)
    p75 = SAColumn('cs_p75', BigIntegerType)
    max = SAColumn('cs_max', BigIntegerType)
    nuniques = SAColumn('cs_nuniques', Integer)

    uvalues = SAColumn('f_uvalues', MutationDict.as_mutable(JSONEncodedObj))
    hist = SAColumn('f_hist', MutationDict.as_mutable(JSONEncodedObj))

    __table_args__ = (
        UniqueConstraint('cs_p_vid', 'cs_c_vid', name='u_cols_stats'),
    )

    def __init__(self, **kwargs):

        for p in self.__mapper__.attrs:
            if p.key in kwargs:

                setattr(self, p.key, kwargs[p.key])
                del kwargs[p.key]

        self.d_vid = str(ObjectNumber.parse(self.p_vid).as_dataset)

        assert str(ObjectNumber.parse(self.p_vid).as_dataset) == str(ObjectNumber.parse(self.c_vid).as_dataset)

    @property
    def dict(self):

        return {
            p.key: getattr(
                self,
                p.key) for p in self.__mapper__.attrs if p.key not in ('data','column', 'table','partition')}

class Column(Base):
    __tablename__ = 'columns'

    vid = SAColumn('c_vid', String(20), primary_key=True)
    id_ = SAColumn('c_id', String(20))
    sequence_id = SAColumn('c_sequence_id', Integer)
    t_vid = SAColumn('c_t_vid',String(20),ForeignKey('tables.t_vid'), nullable=False, index=True)
    d_vid = SAColumn('c_d_vid', String(20), ForeignKey('datasets.d_vid'), nullable=False, index=True)
    t_id = SAColumn('c_t_id', String(20))
    name = SAColumn('c_name', Text)
    altname = SAColumn('c_altname', Text)
    datatype = SAColumn('c_datatype', Text)
    size = SAColumn('c_size', Integer)
    start = SAColumn('c_start', Integer)
    width = SAColumn('c_width', Integer)
    sql = SAColumn('c_sql', Text)
    precision = SAColumn('c_precision', Integer)
    flags = SAColumn('c_flags', Text)
    description = SAColumn('c_description', Text)
    keywords = SAColumn('c_keywords', Text)
    measure = SAColumn('c_measure', Text)
    units = SAColumn('c_units', Text)
    universe = SAColumn('c_universe', Text)
    scale = SAColumn('c_scale', Real)

    # Reference to a column that provides an example of how this column should
    # be used.
    proto_vid = SAColumn('c_proto_vid', String(20), index=True)

    # Reference to a column that this column links to.
    fk_vid = SAColumn('c_fk_vid', String(20), index=True)

    # A column vid, or possibly an equation, describing how this column was
    # created from other columns.
    derivedfrom = SAColumn('c_derivedfrom', String(200))

    data = SAColumn('c_data', MutationDict.as_mutable(JSONEncodedObj))

    is_primary_key = SAColumn('c_is_primary_key', Boolean, default=False)

    unique_constraints = SAColumn('c_unique_constraints', Text)
    indexes = SAColumn('c_indexes', Text)
    uindexes = SAColumn('c_uindexes', Text)
    default = SAColumn('c_default', Text)
    illegal_value = SAColumn('c_illegal_value', Text)

    codes = relationship(Code, backref='column',order_by="asc(Code.key)", cascade="save-update, delete, delete-orphan")

    stats = relationship(ColumnStat, backref='column', cascade="delete, delete-orphan")

    __table_args__ = (
        UniqueConstraint( 'c_sequence_id','c_t_vid', name='_uc_columns_1'),
    )

    DATATYPE_TEXT = 'text'
    DATATYPE_INTEGER = 'integer'
    DATATYPE_INTEGER64 = 'integer64'
    DATATYPE_REAL = 'real'
    DATATYPE_FLOAT = 'float'
    DATATYPE_NUMERIC = 'numeric'
    DATATYPE_DATE = 'date'
    DATATYPE_TIME = 'time'
    DATATYPE_TIMESTAMP = 'timestamp'
    DATATYPE_DATETIME = 'datetime'
    DATATYPE_POINT = 'point'  # Spatalite, sqlite extensions for geo
    DATATYPE_LINESTRING = 'linestring'  # Spatalite, sqlite extensions for geo
    DATATYPE_POLYGON = 'polygon'  # Spatalite, sqlite extensions for geo
    # Spatalite, sqlite extensions for geo
    DATATYPE_MULTIPOLYGON = 'multipolygon'
    DATATYPE_GEOMETRY = 'geometry'  # Spatalite, sqlite extensions for geo
    DATATYPE_CHAR = 'char'
    DATATYPE_VARCHAR = 'varchar'
    DATATYPE_BLOB = 'blob'

    types = {
        # Sqlalchemy, Python, Sql,
        DATATYPE_TEXT: (sqlalchemy.types.Text, str, 'TEXT'),
        DATATYPE_VARCHAR: (sqlalchemy.types.String, str, 'VARCHAR'),
        DATATYPE_CHAR: (sqlalchemy.types.String, str, 'VARCHAR'),
        DATATYPE_INTEGER: (sqlalchemy.types.Integer, int, 'INTEGER'),
        DATATYPE_INTEGER64: (BigIntegerType, long, 'INTEGER64'),
        DATATYPE_REAL: (sqlalchemy.types.Float, float, 'REAL'),
        DATATYPE_FLOAT: (sqlalchemy.types.Float, float, 'REAL'),
        DATATYPE_NUMERIC: (sqlalchemy.types.Float, float, 'REAL'),
        DATATYPE_DATE: (sqlalchemy.types.Date, datetime.date, 'DATE'),
        DATATYPE_TIME: (sqlalchemy.types.Time, datetime.time, 'TIME'),
        DATATYPE_TIMESTAMP: (sqlalchemy.types.DateTime, datetime.datetime, 'TIMESTAMP'),
        DATATYPE_DATETIME: (sqlalchemy.types.DateTime, datetime.datetime, 'DATETIME'),
        DATATYPE_POINT: (GeometryType, str, 'POINT'),
        DATATYPE_LINESTRING: (GeometryType, str, 'LINESTRING'),
        DATATYPE_POLYGON: (GeometryType, str, 'POLYGON'),
        DATATYPE_MULTIPOLYGON: (GeometryType, str, 'MULTIPOLYGON'),
        DATATYPE_GEOMETRY: (GeometryType, str, 'GEOMETRY'),
        DATATYPE_BLOB: (sqlalchemy.types.LargeBinary, buffer, 'BLOB')
    }

    def __init__(self, table=None, **kwargs):

        self.sequence_id = kwargs.get("sequence_id", len(table.columns) + 1 if table else None)

        self.name = kwargs.get("name", None)
        self.altname = kwargs.get("altname", None)
        self.is_primary_key = _clean_flag(kwargs.get("is_primary_key", False))
        self.datatype = kwargs.get("datatype", None)
        self.size = kwargs.get("size", None)
        self.precision = kwargs.get("precision", None)
        self.start = kwargs.get("start", None)
        self.width = kwargs.get("width", None)
        self.sql = kwargs.get("sql", None)
        self.flags = kwargs.get("flags", None)
        self.description = kwargs.get("description", None)
        self.keywords = kwargs.get("keywords", None)
        self.measure = kwargs.get("measure", None)
        self.units = kwargs.get("units", None)
        self.universe = kwargs.get("universe", None)
        self.scale = kwargs.get("scale", None)
        self.fk_vid = kwargs.get("fk_vid", kwargs.get("foreign_key", None))
        self.proto_vid = kwargs.get("proto_vid", kwargs.get("proto", None))
        self.derivedfrom = kwargs.get("derivedfrom", None)
        self.data = kwargs.get("data", None)

        # the table_name attribute is not stored. It is only for
        # building the schema, linking the columns to tables.
        self.table_name = kwargs.get("table_name", None)

        assert self.sequence_id is not None

        if not self.name:
            self.name = 'column' + str(self.sequence_id)
            # raise ValueError('Column must have a name. Got: {}'.format(kwargs))

        # Don't allow these values to be the empty string
        self.fk_vid = self.fk_vid or None
        self.proto_vid = self.proto_vid or None
        self.derivedfrom = self.derivedfrom or None

        if table:
            self.t_id = table.id_
            self.t_vid = table.vid
            ton = ObjectNumber.parse(table.vid)
            con = ColumnNumber(ton, self.sequence_id)
            self.vid = str(con)
            self.id = str(con.rev(None))

    def type_is_int(self):
        return self.datatype in (
            Column.DATATYPE_INTEGER,
            Column.DATATYPE_INTEGER64)

    def type_is_text(self):
        return self.datatype in (
            Column.DATATYPE_TEXT,
            Column.DATATYPE_CHAR,
            Column.DATATYPE_VARCHAR)

    def type_is_geo(self):
        return self.datatype in (
            Column.DATATYPE_POINT,
            Column.DATATYPE_LINESTRING,
            Column.DATATYPE_POLYGON,
            Column.DATATYPE_MULTIPOLYGON,
            Column.DATATYPE_GEOMETRY)

    def type_is_gvid(self):
        return 'gvid' in self.name

    def type_is_time(self):
        return self.datatype in (
            Column.DATATYPE_TIME,
            Column.DATATYPE_TIMESTAMP,
            Column.DATATYPE_DATETIME,
            Column.DATATYPE_DATE)

    @property
    def sqlalchemy_type(self):
        return self.types[self.datatype][0]

    @property
    def python_type(self):
        return self.types[self.datatype][1]

    def python_cast(self, v):
        """Cast a value to the type of the column.

        Primarily used to check that a value is valid; it will throw an
        exception otherwise

        """

        if self.type_is_time():
            import dateutil.parser
            dt = dateutil.parser.parse(v)

            if self.datatype == Column.DATATYPE_TIME:
                dt = dt.time()
            if not isinstance(dt, self.python_type):
                raise TypeError(
                    '{} was parsed to {}, expected {}'.format(
                        v,
                        type(dt),
                        self.python_type))

            return dt
        else:
            # This isn't calling the python_type method -- it's getting a python type, then instantialting it,
            # such as "int(v)"
            return self.python_type(v)

    @property
    def schema_type(self):

        if not self.datatype:
            from dbexceptions import ConfigurationError
            raise ConfigurationError("Column '{}' has no datatype".format(self.name))

        try:
            return self.types[self.datatype][2]
        except KeyError:

            raise

    @classmethod
    def convert_numpy_type(cls, dtype):
        """Convert a numpy dtype into a Column datatype. Only handles common
        types.

        Implemented as a function to decouple from numpy

        """

        import numpy as np

        m = {
            'int64': cls.DATATYPE_INTEGER64,
            'float64': cls.DATATYPE_FLOAT,
            'object': cls.DATATYPE_TEXT  # Hack. Pandas makes strings into object.

        }

        t = m.get(dtype.name, None)

        if not t:
            raise TypeError(
                "Failed to convert numpy type: '{}' ".format(
                    dtype.name))

        return t

    @classmethod
    def convert_python_type(cls, py_type_in, name=None):

        type_map = {
            unicode: str
        }

        for col_type, (sla_type, py_type, sql_type) in cls.types.items():
            if py_type == type_map.get(py_type_in, py_type_in):
                if col_type == 'blob' and name and name.endswith('geometry'):
                    return cls.DATATYPE_GEOMETRY
                else:
                    return col_type

        return None

    @property
    def foreign_key(self):
        return self.fk_vid

    @property
    def dict(self):
        """A dict that holds key/values for all of the properties in the
        object.

        :return:

        """
        d = {p.key: getattr(self,p.key) for p in self.__mapper__.attrs
             if p.key not in ('table','stats','_codes', 'data')}

        if not d:
            raise Exception(self.__dict__)

        d['schema_type'] = self.schema_type

        if self.data:
            # Copy data fields into top level dict, but don't overwrite existind values.
            for k, v in self.data.items() :
                if k not in d and k not in ('table','stats','_codes', 'data'):
                    d[k] = v

        return d

    @property
    def nonull_dict(self):
        """Like dict, but does not hold any null values.

        :return:

        """
        return {k: v for k, v in self.dict.items() if v and k != '_codes'}

    @property
    def insertable_dict(self):
        """Like dict, but properties have the table prefix, so it can be
        inserted into a row."""

        d = {p.key: getattr(self, p.key) for p in self.__mapper__.attrs if p.key not in ('table', 'stats', '_codes')}

        x = {('c_' + k).strip('_'): v for k, v in d.items()}

        return x

    @staticmethod
    def mangle_name(name):
        """Mangles a column name to a standard form, remoing illegal
        characters.

        :param name:
        :return:

        """
        import re
        try:
            return re.sub('_+','_',re.sub('[^\w_]','_',name).lower()).rstrip('_')
        except TypeError:
            raise TypeError(
                'Trying to mangle name with invalid type of: ' + str(type(name)))

    @property
    def fq_name(self):
        """Fully Qualified Name.

        A column Name with the column id as a prefix

        """
        return "{}_{}".format(self.id_, self.name)

    @property
    @memoize
    def reverse_code_map(self):
        """Return a map from a code ( usually a string ) to the  shorter numeric value"""

        return { c.value:(c.ikey if c.ikey else c.key) for c in self.codes}

    @property
    @memoize
    def forward_code_map(self):
        """Return  a map from the short code to the full value """

        return { c.key:c.value for c in self.codes}

    def add_code(self, key, value, description=None, data=None):
        """

        :param key: The code value that appears in the datasets, either a string or an int
        :param value: The string value the key is mapped to
        :param description:  A more detailed description of the code
        :param data: A data dict to add to the ORM record
        :return: the code record
        """
        from sqlalchemy.orm.session import Session

        # Ignore codes we already have, but will not catch codes added earlier for this same
        # object, since the code are cached

        for cd in self.codes:
            if cd.key == str(key):
                return cd

        def cast_to_int(s):
            try:
                return int(s)
            except ValueError:
                return None

        cd = Code(c_vid=self.vid, t_vid=self.t_vid,
                  key=str(key),
                  ikey= cast_to_int(key),
                  value=value,
                  description=description, data=data)

        self.codes.append(cd)

        return cd

    @staticmethod
    def before_insert(mapper, conn, target):
        """event.listen method for Sqlalchemy to set the seqience_id for this
        object and create an ObjectNumber value for the id_"""

        #from identity import ObjectNumber
        #assert not target.fk_vid or not ObjectNumber.parse(target.fk_vid).revision

        if target.sequence_id is None:
            # In case this happens in multi-process mode
            conn.execute("BEGIN IMMEDIATE")
            sql = text(
                '''SELECT max(c_sequence_id)+1 FROM columns WHERE c_t_id = :tid''')

            max_id, = conn.execute(sql, tid=target.t_id).fetchone()

            if not max_id:
                max_id = 1

            target.sequence_id = max_id

        Column.before_update(mapper, conn, target)

    @staticmethod
    def before_update(mapper, conn, target):
        """Set the column id number based on the table number and the sequence
        id for the column."""

        if target.id_ is None:

            if target.table:
                table_on = ObjectNumber.parse(target.table.vid)
            else:
                table_on = ObjectNumber.parse(target.t_vid)

            if not target.vid:
                target.vid = str(ColumnNumber(table_on, target.sequence_id))

            if not target.id_:
                target.id_ = str(ColumnNumber(table_on, target.sequence_id).rev(None))

        target.d_vid = str(ObjectNumber.parse(target.t_vid).as_dataset)

    def __repr__(self):
        return "<column: {}, {}>".format(self.name, self.vid)

event.listen(Column, 'before_insert', Column.before_insert)
event.listen(Column, 'before_update', Column.before_update)

class Table(Base, LinkableMixin, DataPropertyMixin):
    __tablename__ = 'tables'

    vid = SAColumn('t_vid', String(20), primary_key=True)
    id_ = SAColumn('t_id', String(20), primary_key=False)
    d_id = SAColumn('t_d_id', String(20))
    d_vid = SAColumn('t_d_vid',String(20),ForeignKey('datasets.d_vid'),index=True)
    # This is a freign key, but is not declared as such
    p_vid = SAColumn('t_p_vid', String(20), index=True, nullable=True)
    sequence_id = SAColumn('t_sequence_id', Integer, nullable=False)
    name = SAColumn('t_name', String(200), nullable=False)
    altname = SAColumn('t_altname', Text)
    description = SAColumn('t_description', Text)
    universe = SAColumn('t_universe', String(200))
    keywords = SAColumn('t_keywords', Text)
    type = SAColumn('t_type', String(20))
    # Reference to a column that provides an example of how this table should be used.
    proto_vid = SAColumn('t_proto_vid', String(20), index=True)

    installed = SAColumn('t_installed', String(100))
    data = SAColumn('t_data', MutationDict.as_mutable(JSONEncodedObj))

    __table_args__ = (
        #ForeignKeyConstraint([d_vid, d_location], ['datasets.d_vid', 'datasets.d_location']),
        UniqueConstraint('t_sequence_id', 't_d_vid', name='_uc_tables_1'),
        UniqueConstraint('t_name', 't_d_vid', name='_uc_tables_2'),
    )

    columns = relationship(Column, backref='table', order_by="asc(Column.sequence_id)",
                           cascade="merge, delete, delete-orphan", lazy='joined')

    def __init__(self, dataset, **kwargs):

        assert 'proto' not in kwargs

        self.sequence_id = kwargs.get("sequence_id", None)
        self.name = kwargs.get("name", None)
        self.vname = kwargs.get("vname", None)
        self.altname = kwargs.get("altname", None)
        self.description = kwargs.get("description", None)
        self.universe = kwargs.get("universe", None)
        self.keywords = kwargs.get("keywords", None)
        self.type = kwargs.get("type", 'table')
        self.proto_vid = kwargs.get("proto_vid")
        self.data = kwargs.get("data", None)

        self.d_id = dataset.id_
        self.d_vid = dataset.vid

        don = ObjectNumber.parse(dataset.vid)
        ton = TableNumber(don, self.sequence_id)

        self.vid = str(ton)
        self.id_ = str(ton.rev(None))

        if self.name:
            self.name = self.mangle_name(self.name, kwargs.get('preserve_case',False))

        self.init_on_load()

    @property
    def dict(self):
        d = {
            k: v for k,
            v in self.__dict__.items() if k in [
                'id_','vid','d_id','d_vid','sequence_id','name','altname','vname','description','universe','keywords',
                'installed','proto_vid','type','codes']}

        if self.data:
            for k in self.data:
                assert k not in d, "Value '{}' is a table field and should not be in data ".format(k)
                d[k] = self.data[k]

        d['is_geo'] = False

        for c in self.columns:
            if c in ('geometry', 'wkt', 'wkb', 'lat'):
                d['is_geo'] = True

        d['foreign_indexes'] =  list(set([c.data['index'].split(":")[0] for c in self.columns if c.data.get('index',False)]))

        return d

    @property
    def nonull_dict(self):
        return {k: v for k, v in self.dict.items() if v and k not in 'codes'}

    @property
    def nonull_col_dict(self):

        tdc = {}

        for c in self.columns:
            tdc[c.id_] = c.nonull_dict
            tdc[c.id_]['codes'] = {cd.key: cd.dict for cd in c.codes}

        td = self.nonull_dict
        td['columns'] = tdc

        return td

    def link_columns(self, other):
        """Return columns that can be used to link another table to this one"""

        def protos(t):
            from identity import ObjectNumber

            protos = {}

            protos.update({ c.fk_vid:c for c in t.columns if c.fk_vid })
            protos.update({ c.proto_vid:c for c in t.columns if c.proto_vid})

            # HACK: The numbering in the proto dataset changes, so we have to make substitutions
            if 'c00104002' in protos:
                protos['c00109003'] = protos['c00104002']
                del protos['c00104002']

            protos = { str(ObjectNumber.parse(n).rev(None)):c for n, c in protos.items() } # Remove revisions

            return protos

        protos_s = protos(self)
        protos_o = protos(other)

        inter =  set(protos_s.keys())  & set(protos_o.keys())

        return list(set( (protos_s[n], protos_o[n])  for n in inter ) )

    @property
    def insertable_dict(self):
        x = {('t_' + k).strip('_'): v for k, v in self.dict.items()}

        if not 't_vid' in x or not x['t_vid']:
            raise ValueError("Must have vid set: {} ".format(x))

        return x

    # For linking tables to manifests
    @property
    def linked_files(self):
        return self._get_link_array('files', File, File.ref)

    def link_file(self, f):
        return self._append_link('files', f.ref)

    def delink_file(self, f):
        return self._remove_link('files', f.ref)

    @property
    def info(self):

        x =  """
------ Table: {name} ------
id   : {id_}
vid  : {vid}
name : {name}
Columns:
""".format(**self.dict)

        for c in self.columns:
            # ['id','vid','sequence_id', 't_vid', 'name', 'description', 'keywords', 'datatype', 'size', 'is_primary_kay', 'data']}

            x += "   {sequence_id:3d} {name:12s} {schema_type:8s} {description}\n".format(
                **c.dict)

        return x

    def _repr_html_(self):
        """IPython display."""

        t1 = """
        <table>
        <tr><th>Name</th><td>{name}</td></tr>
        <tr><th>Id</th><td>{id_}</td></tr>
        <tr><th>Vid</th><td>{vid}</td></tr>
        </table>
        """.format(**self.dict)

        return t1 + self.html_table()

    def html_table(self):
        """"""

        rows = []
        rows.append(
            "<tr><th>#</th><th>Name</th><th>Datatype</th><th>description</th></tr>")
        for c in self.columns:
            rows.append(
                "<tr><td>{sequence_id:d}</td><td>{name:s}</td><td>{schema_type:s}</td><td>{description}</td></tr>".format(**c.dict))

        return "<table>\n" + "\n".join(rows) + "\n</table>"

    def vid_select(self):
        """Return a SQL fragment to translate the column names to vids.

        This allows the identity of the column to propagate through
        views.

        """

        cols = []
        raise DeprecationWarning()
        return ",".join(["{} AS {}".format(c.name, c.vid)
                        for c in self.columns])

    @orm.reconstructor
    def init_on_load(self):
        self._or_validator = None
        self._and_validator = None
        self._null_row = None
        self._row_hasher = None

    @staticmethod
    def before_insert(mapper, conn, target):
        """event.listen method for Sqlalchemy to set the seqience_id for this
        object and create an ObjectNumber value for the id_"""
        if target.sequence_id is None:
            sql = text(
                '''SELECT max(t_sequence_id)+1 FROM tables WHERE t_d_id = :did''')

            max_id, = conn.execute(sql, did=target.d_id).fetchone()

            if not max_id:
                max_id = 1

            target.sequence_id = max_id

        Table.before_update(mapper, conn, target)

    @staticmethod
    def before_update(mapper, conn, target):
        """Set the Table ID based on the dataset number and the sequence number
        for the table."""
        if isinstance(target, Column):
            raise TypeError('Got a column instead of a table')

        if target.id_ is None:
            dataset_id = ObjectNumber.parse(target.d_id)
            target.id_ = str(TableNumber(dataset_id, target.sequence_id))

    @staticmethod
    def mangle_name(name, preserve_case=False):
        import re
        try:
            r = re.sub('[^\w_]', '_', name.strip())

            if not preserve_case:
                r = r.lower()

            return r
        except TypeError:
            raise TypeError('Not a valid type for name ' + str(type(name)))

    @property
    def oid(self):
        return TableNumber(self.d_id, self.sequence_id)

    def add_column(self, name, **kwargs):
        """Add a column to the table, or update an existing one."""

        import sqlalchemy.orm.session
        from dbexceptions import NotFoundError

        s = sqlalchemy.orm.session.Session.object_session(self)

        assert s, "Can't create column with this method unless the table has a session"

        name = Column.mangle_name(name)

        if not kwargs.get('fast', False):
            try:
                row = self.column(name)
            except NotFoundError:
                row = None
        else:
            row = None

        if row:
            extant = True

        else:
            row = Column(self, name=name, **kwargs)
            extant = False

        if kwargs.get('data', False):
            row.data = dict(row.data.items() + kwargs['data'].items())

        for key, value in kwargs.items():

            excludes = ['d_id', 't_id', 'name', 'schema_type', 'data']

            # Proto is the name of the object.
            if key == 'proto' and isinstance(value, basestring):
                key = 'proto_vid'

            if extant:
                excludes.append('sequence_id')

            if key[0] != '_' and key not in excludes:
                try:
                    setattr(row, key, value)
                except AttributeError:
                    raise AttributeError(
                        "Column record has no attribute {}".format(key))

            if isinstance(value, basestring) and len(value) == 0:
                if key == 'is_primary_key':
                    value = False
                    setattr(row, key, value)

        # If the id column has a description and the table does not, add it to
        # the table.
        if row.name == 'id' and row.is_primary_key and not self.description:
            self.description = row.description
            s.merge(self)

        if extant:
            row = s.merge(row)
        else:
            s.add(row)

        if kwargs.get('commit', True):
            s.commit()

        return row

    def add_id_column(self):
        self.add_column(name='id',datatype='integer',is_primary_key = True, description = self.description)

    def column(self, ref, default=None):
        from dbexceptions import NotFoundError

        # AFAIK, all of the columns in the relationship will get loaded if any one is accessed,
        # so iterating over the collection only involved one SELECT.

        for c in self.columns:
            if str(ref) == c.name or str(ref) == c.id_ or str(ref) == c.vid:
                return c

        raise NotFoundError("Failed to find column '{}' in table '{}' for ref: '{}' "
                            .format(ref,self.name, ref))

    @property
    def primary_key(self):
        for c in self.columns:
            if c.is_primary_key:
                return c
        return None

    def get_fixed_regex(self):
        """Using the size values for the columns for the table, construct a
        regular expression to  parsing a fixed width file."""
        import re

        pos = 0
        regex = ''
        header = []

        for col in self.columns:

            size = col.width if col.width else col.size

            if not size:
                continue

            pos += size

            regex += "(.{{{}}})".format(size)
            header.append(col.name)

        return header, re.compile(regex), regex

    def get_fixed_unpack(self):
        """Using the size values for the columns for the table, construct a
        regular expression to  parsing a fixed width file."""
        from functools import partial
        import struct
        unpack_str = ''
        header = []
        length = 0

        for col in self.columns:

            size = col.width

            if not size:
                continue

            length += size

            unpack_str += "{}s".format(size)

            header.append(col.name)

        return partial(struct.unpack, unpack_str), header, unpack_str, length

    def get_fixed_colspec(self):
        """Return the column specification suitable for use in  the Panads
        read_fwf function.

        This will ignore any columns that don't have one or both of the
        start and width values

        """

        # Warning! Assuming th start values are sorted. Really should check.

        return (
            [c.name for c in self.columns if c.start and c.width],
            [(c.start, c.start + c.width) for c in self.columns if c.start and c.width]
        )

    @property
    def null_row(self):
        if self._null_row is None:
            self._null_row = []
            for col in self.columns:
                if col.is_primary_key:
                    v = None
                elif col.default:
                    v = col.default
                else:
                    v = None

                self._null_row.append(v)

        return self._null_row

    @property
    def null_dict(self):
        if self._null_row is None:
            self._null_row = {}
            for col in self.columns:
                if col.is_primary_key:
                    v = None
                elif col.default:
                    v = col.default
                else:
                    v = None

                self._null_row[col.name] = v

        return self._null_row

    @property
    def header(self):
        """Return an array of column names in the same order as the column
        definitions, to be used zip with a row when reading a CSV file.

        >> row = dict(zip(table.header, row))

        """

        return [c.name for c in self.columns]

    def _get_validator(self, and_join=True):
        """Return a lambda function that, when given a row to this table,
        returns true or false to indicate the validitity of the row.

        :param and_join: If true, join multiple column validators with AND, other
        wise, OR
        :type and_join: Bool

        :rtype: a `LibraryDb` object

        """

        f = prior = lambda row: True
        first = True
        for i, col in enumerate(self.columns):

            if col.data.get('mandatory', False):
                default_value = col.default
                index = i

                if and_join:
                    f = lambda row, default_value=default_value, index=index, prior=prior: prior(
                        row) and str(row[index]) != str(default_value)
                elif first:
                    # OR joins would either need the initial F to be 'false',
                    # or just don't use it
                    f = lambda row, default_value=default_value, index=index: str(
                        row[index]) != str(default_value)
                else:
                    f = lambda row, default_value=default_value, index=index, prior=prior: prior(
                        row) or str(row[index]) != str(default_value)

                prior = f
                first = False

        return f

    def validate_or(self, values):

        if self._or_validator is None:
            self._or_validator = self._get_validator(and_join=False)

        return self._or_validator(values)

    def validate_and(self, values):

        if self._and_validator is None:
            self._and_validator = self._get_validator(and_join=True)

        return self._and_validator(values)

    def _get_hasher(self):
        """Return a  function to generate a hash for the row."""
        import hashlib

        # Try making the hash set from the columns marked 'hash'
        indexes = [i for i, c in enumerate(self.columns) if
                   c.data.get('hash', False) and not c.is_primary_key]

        # Otherwise, just use everything by the primary key.
        if len(indexes) == 0:
            indexes = [
                i for i,
                c in enumerate(
                    self.columns) if not c.is_primary_key]

        def hasher(values):
            m = hashlib.md5()
            for index in indexes:
                x = values[index]
                try:
                    m.update(
                        x.encode('utf-8') +
                        '|')  # '|' is so 1,23,4 and 12,3,4 aren't the same
                except:
                    m.update(str(x) + '|')
            return int(m.hexdigest()[:14], 16)

        return hasher

    def row_hash(self, values):
        """Calculate a hash from a database row."""

        if self._row_hasher is None:
            self._row_hasher = self._get_hasher()

        return self._row_hasher(values)

    @property
    def caster(self):
        """Returns a function that takes a row that can be indexed by positions
        which returns a new row with all of the values cast to schema types."""
        from ambry.transform import CasterTransformBuilder

        bdr = CasterTransformBuilder()

        for c in self.columns:
            bdr.append(c.name, c.python_type)

        return bdr

    def add_installed_name(self, name):
        self._append_string_to_list('installed_names', name)

    @property
    def linked_manifests(self):
        return self._get_link_array('manifests', File, File.ref)

    def link_manifest(self, f):
        return self._append_link('manifests', f.ref)

    def delink_manifest(self, f):
        return self._remove_link('manifests', f.ref)

event.listen(Table, 'before_insert', Table.before_insert)
event.listen(Table, 'before_update', Table.before_update)

class Config(Base):

    __tablename__ = 'config'

    d_vid = SAColumn('co_d_vid', String(16), primary_key=True)
    group = SAColumn('co_group', String(200), primary_key=True)
    key = SAColumn('co_key', String(200), primary_key=True)

    value = SAColumn('co_value', JSONAlchemy(Text()))

    source = SAColumn('co_source', String(200))

    def __init__(self, **kwargs):
        self.d_vid = kwargs.get("d_vid", None)
        self.group = kwargs.get("group", None)
        self.key = kwargs.get("key", None)
        self.value = kwargs.get("value", None)
        self.source = kwargs.get("source", None)

    @property
    def dict(self):
        return {p.key: getattr(self, p.key) for p in self.__mapper__.attrs}

    def __repr__(self):
        return "<config: {},{},{} = {}>".format(self.d_vid,self.group,self.key,self.value)

class Partition(Base, LinkableMixin):
    __tablename__ = 'partitions'

    vid = SAColumn('p_vid', String(20), primary_key=True, nullable=False)
    id_ = SAColumn('p_id', String(20), nullable=False)
    name = SAColumn('p_name', String(200), nullable=False, index=True)
    vname = SAColumn('p_vname',String(200),unique=True,nullable=False,index=True)
    fqname = SAColumn('p_fqname',String(200),unique=True,nullable=False,index=True)
    cache_key = SAColumn('p_cache_key',String(200),unique=True,nullable=False,index=True)
    sequence_id = SAColumn('p_sequence_id', Integer)
    t_vid = SAColumn('p_t_vid',String(20),ForeignKey('tables.t_vid'),nullable=False,index=True)
    t_id = SAColumn('p_t_id', String(20))
    d_vid = SAColumn('p_d_vid',String(20),ForeignKey('datasets.d_vid'),nullable=False,index=True)
    d_id = SAColumn('p_d_id', String(20))
    ref = SAColumn('p_ref', String(200), index=True)
    time = SAColumn('p_time', String(20))
    space = SAColumn('p_space', String(50))
    grain = SAColumn('p_grain', String(50))
    variant = SAColumn('p_variant', String(50))
    format = SAColumn('p_format', String(50))
    segment = SAColumn('p_segment', Integer)
    min_key = SAColumn('p_min_key', BigIntegerType)
    max_key = SAColumn('p_max_key', BigIntegerType)
    count = SAColumn('p_count', Integer)
    state = SAColumn('p_state', String(50))
    data = SAColumn('p_data', MutationDict.as_mutable(JSONEncodedObj))

    installed = SAColumn('p_installed', String(100))

    __table_args__ = (#ForeignKeyConstraint( [d_vid, d_location], ['datasets.d_vid','datasets.d_location']),
        UniqueConstraint('p_sequence_id', 'p_t_vid', name='_uc_partitions_1'),
    )

    # For the primary table for the partition. There is one per partition, but a table
    # can be primary in multiple partitions.
    table = relationship('Table', backref='partitions', foreign_keys='Partition.t_vid')

    warehouse_tables = relationship('Table', backref='source_partition', foreign_keys='Table.p_vid',
                                    primaryjoin="Partition.vid == Table.p_vid")

    stats = relationship(ColumnStat, backref='partition', cascade="delete, delete-orphan")

    def __init__(self, dataset, t_id, **kwargs):

        self.vid = kwargs.get("vid", kwargs.get("id_", None))
        self.id_ = kwargs.get("id", kwargs.get("id_", None))
        self.name = kwargs.get("name", kwargs.get("name", None))
        self.vname = kwargs.get("vname", None)
        self.ref = kwargs.get("ref", None)
        self.fqname = kwargs.get("fqname", None)
        self.cache_key = kwargs.get("cache_key", None)
        self.sequence_id = kwargs.get("sequence_id", None)

        self.space = kwargs.get("space", None)
        self.time = kwargs.get("time", None)
        self.grain = kwargs.get('grain', None)
        self.format = kwargs.get('format', None)
        self.segment = kwargs.get('segment', None)
        self.data = kwargs.get('data', None)

        self.d_vid = dataset.vid
        self.d_id = dataset.id_

        self.t_id = t_id

        tables = { t.id_: t.name for t in dataset.tables }

        don = ObjectNumber.parse(self.d_vid)
        ton = ObjectNumber.parse(self.t_id)
        self.t_vid = str(ton.rev(don.revision))

        from identity import PartialPartitionName
        kwargs['table'] = tables[self.t_id]
        ppn = PartialPartitionName(**kwargs)

        if not self.vname:
            self.vname = ppn.promote(dataset.identity.name).vname
            self.name = ppn.promote(dataset.identity.name).name

        if not self.cache_key:
            self.cache_key = ppn.promote(dataset.identity.name).cache_key

        assert self.cache_key is not None

        if True:  # Debugging
            from partition import extension_for_format_name

            ext = extension_for_format_name(self.format)

            assert self.cache_key.endswith(ext)

    @property
    def identity(self):
        """Return this partition information as a PartitionId."""
        from sqlalchemy.orm import object_session
        from identity import PartitionIdentity

        if self.dataset is None:
            # The relationship will be null until the object is committed
            s = object_session(self)

            ds = s.query(Dataset).filter(Dataset.id_ == self.d_id).one()
        else:
            ds = self.dataset

        d = {
            'id': self.id_,
            'vid': self.vid,
            'name': self.name,
            'vname': self.vname,
            'ref': self.ref,
            'space': self.space,
            'time': self.time,
            'table': self.table.name if self.t_vid is not None else None,
            'grain': self.grain,
            'segment': self.segment,
            'format': self.format if self.format else 'db'
        }

        return PartitionIdentity.from_dict(dict(ds.dict.items() + d.items()))

    @property
    def dict(self):
        from geoid.civick import GVid

        d = {
            'id': self.id_,
            'sequence_id': self.sequence_id,
            'vid': self.vid,
            'name': self.name,
            'vname': self.vname,
            'ref': self.ref,
            'fqname': self.fqname,
            'cache_key': self.cache_key,
            'd_id': self.d_id,
            'd_vid': self. d_vid,
            't_id': self.t_id,
            't_vid': self. t_vid,
            'space': self.space,
            'time': self.time,
            'table': self.table.name if self.t_vid is not None else None,
            'table_vid': self.t_vid,
            'grain': self.grain,
            'segment': self.segment,
            'format': self.format if self.format else 'db',
            'count': self.count,
            'min_key': self.min_key,
            'max_key': self.max_key
        }

        for k in self.data:
            assert k not in d
            d[k] = self.data[k]

        d['dataset'] = self.dataset.dict

        d['colstats'] = {s.column.id_: s.dict for s in self._stats}

        if 'geo_grain' in d:
            d['geo_grain'] = {
                'vids': d['geo_grain'],
                'names': []
            }

            for gvid_str in d['geo_grain']['vids']:
                try:
                    gvid = GVid.parse(gvid_str)
                    d['geo_grain']['names'].append(gvid.level.title())
                except KeyError:
                    d['geo_grain']['names'].append(gvid_str)

        if 'geo_coverage' in d:
            d['geo_coverage'] = {
                'vids': d['geo_coverage'],
                'names': []
            }

        if 'time_coverage' in d:
            from util.datestimes import expand_to_years

            all_years = expand_to_years(d['time_coverage'])

            d['time_coverage'] = {
                'years': d['time_coverage'],
                'min': min(all_years) if all_years else None,
                'max': max(all_years) if all_years else None,
            }

        d['foreign_indexes'] = list(set([c.data['index'].split(':')[0] for c in self.table.columns
                                 if c.data.get('index', False)]))

        return d

    @property
    def nonull_dict(self):
        d = {k: v for k, v in self.dict.items() if v}

        d['format'] = self.format if self.format else 'db'
        d['table'] = self.table.name if self.t_vid is not None else None

        return d

    @property
    def insertable_dict(self):
        return {('p_' + k).strip('_'): v for k, v in self.dict.items()}

    def __repr__(self):
        return "<{} partition: {}>".format(self.format, self.vname)

    def set_ids(self, sequence_id):
        from identity import Identity

        if not self.vid or not self.id_:

            self.sequence_id = sequence_id

            don = ObjectNumber.parse(self.d_vid)
            pon = PartitionNumber(don, self.sequence_id)

            self.vid = str(pon)
            self.id_ = str(pon.rev(None))

        self.fqname = Identity._compose_fqname(self.vname, self.vid)

    # For linking partitions to manifests
    @property
    def linked_manifests(self):
        return self._get_link_array('manifests', File, File.ref)

    def link_manifest(self, f):
        return self._append_link('manifests', f.ref)

    def delink_manifest(self, f):
        return self._remove_link('manifests', f.ref)

    @property
    def linked_stores(self):
        return self._get_link_array('stores', File, File.ref)

    def link_store(self, f):
        return self._append_link('stores', f.ref)

    def delink_store(self, f):
        return self._remove_link('stores', f.ref)

    def add_stat(self, c_vid, stats):
        """Add a statistics records for a column of a table in the partition.

        :param c_vid: The column vid.
        :param stats:  A dict of stats values. See the code for which values are valid.
        :return:

        """

        from sqlalchemy.orm.session import Session

        # Names that come from the Pandas describe() method
        stat_map = {'25%': 'p25', '50%': 'p50', '75%': 'p75'}

        stats = {stat_map.get(k, k): v for k, v in stats.items()}

        cs = ColumnStat(p_vid=self.vid, c_vid=c_vid, **stats)

        self._stats.append(cs)

        return cs

    @property
    def stats(self):

        class Bunch(object):
            """Dict and object access to properties"""
            def __init__(self, o):
                self.__dict__.update(o)

            def __str__(self):
                return str(self.__dict__)

            def __repr__(self):
                return str(self.__dict__)

            def items(self):
                return self.__dict__.items()

        cols = {s.column.name: Bunch(s.dict) for s in self._stats}

        return Bunch(cols)

    @staticmethod
    def before_insert(mapper, conn, target):
        """event.listen method for Sqlalchemy to set the sequence for this
        object and create an ObjectNumber value for the id_"""
        from identity import Identity

        if target.sequence_id is None:
            # These records can be added in an multi-process environment, we
            # we need exclusive locking here, where we don't for other sequence
            # ids.
            conn.execute("BEGIN IMMEDIATE")
            sql = text("SELECT max(p_sequence_id)+1 FROM Partitions WHERE p_d_id = :did")

            max_id, = conn.execute(sql, did=target.d_id).fetchone()

            if not max_id:
                max_id = 1

            target.sequence_id = max_id

        target.set_ids(target.sequence_id)

        Partition.before_update(mapper, conn, target)

    @staticmethod
    def before_update(mapper, conn, target):
        """Set the column id number based on the table number and the sequence
        id for the column."""
        if not target.id_:
            dataset = ObjectNumber.parse(target.d_id)
            target.id_ = str(PartitionNumber(dataset, target.sequence_id))

    @staticmethod
    def set_t_vid(target, value, oldvalue, initiator):
        "Check that t_vid isn't set to Null"
        if not bool(value):
            raise AssertionError("Partition.t_vid can't be null (set_t_vid): {} -> {}".format(oldvalue, value))

#event.listen(Partition.t_vid, 'set', Partition.set_t_vid)
event.listen(Partition, 'before_insert', Partition.before_insert)
event.listen(Partition, 'before_update', Partition.before_update)

class File(Base, SavableMixin, LinkableMixin):
    __tablename__ = 'files'

    TYPE = Constant()
    TYPE.BUNDLE = LocationRef.LOCATION.LIBRARY
    TYPE.PARTITION = LocationRef.LOCATION.PARTITION
    TYPE.SOURCE = LocationRef.LOCATION.SOURCE
    TYPE.SREPO = LocationRef.LOCATION.SREPO
    TYPE.UPSTREAM = LocationRef.LOCATION.UPSTREAM
    TYPE.REMOTE = LocationRef.LOCATION.REMOTE
    TYPE.REMOTEPARTITION = LocationRef.LOCATION.REMOTEPARTITION

    TYPE.MANIFEST = 'manifest'
    TYPE.DOC = 'doc'
    TYPE.EXTRACT = 'extract'
    TYPE.STORE = 'store'
    TYPE.DOWNLOAD = 'download'

    PROCESS = Constant()
    PROCESS.MODIFIED = 'modified'
    PROCESS.UNMODIFIED = 'unmodified'
    PROCESS.DOWNLOADED = 'downloaded'
    PROCESS.CACHED = 'cached'

    oid = SAColumn('f_id', Integer, primary_key=True, nullable=False)
    path = SAColumn('f_path', Text, nullable=False)
    ref = SAColumn('f_ref', Text, index=True, nullable=False)
    type_ = SAColumn('f_type', Text, nullable=False)
    source_url = SAColumn('f_source_url', Text, nullable=False)
    process = SAColumn('f_process', Text)
    state = SAColumn('f_state', Text)
    hash = SAColumn('f_hash', Text)
    modified = SAColumn('f_modified', Integer)
    size = SAColumn('f_size', BigIntegerType)

    priority = SAColumn('f_priority', Integer)

    data = SAColumn('f_data', MutationDict.as_mutable(JSONEncodedObj))

    content = SAColumn('f_content', Binary)

    __table_args__ = (
        UniqueConstraint('f_ref', 'f_type', 'f_source_url', name='u_ref_path'),
    )

    def __init__(self, **kwargs):

        if not kwargs.get("ref", None):
            import hashlib

            kwargs['ref'] = hashlib.md5(kwargs['path']).hexdigest()

        self.oid = kwargs.get("oid", None)
        self.path = kwargs.get("path", None)
        self.source_url = kwargs.get("source_url", kwargs.get("source", None))
        self.process = kwargs.get("process", None)
        self.state = kwargs.get("state", None)
        self.modified = kwargs.get("modified", None)
        self.size = kwargs.get("size", None)
        self.group = kwargs.get("group", None)
        self.ref = kwargs.get("ref", None)
        self.hash = kwargs.get("hash", None)
        self.type_ = kwargs.get("type", kwargs.get("type_", None))

        self.data = kwargs.get('data', None)
        self.priority = kwargs.get('priority', 0)
        self.content = kwargs.get('content', None)

    def __repr__(self):
        return "<file: {}; {}>".format(self.path, self.ref, self.state)

    def update(self, f):
        """Copy another files properties into this one."""

        for p in self.__mapper__.attrs:

            if p.key == 'oid':
                continue
            try:
                setattr(self, p.key, getattr(f, p.key))

            except AttributeError:
                # The dict() method copies data property values into the main dict,
                # and these don't have associated class properties.
                continue

    @property
    def dict(self):

        d = dict((col, getattr(self,col)) for col in [
                'oid','path','ref','type_','source_url','process',
                'state','hash','modified','size', 'priority'])

        if self.data:
            for k in self.data:
                assert k not in d
                d[k] = self.data[k]

        return d

    @property
    def record_dict(self):
        """Like dict, but does not move data items into the top level."""
        return {p.key: getattr(self, p.key) for p in self.__mapper__.attrs}

    @property
    def insertable_dict(self):
        """Like record_dict, but prefixes all of the keys with 'f_', so it can
        be used in inserts."""
        # .strip('_') is for type_
        return {
            'f_' +
            p.key.strip('_'): getattr(
                self,
                p.key) for p in self.__mapper__.attrs}

    # These partition and file acessors were originally implemented as many-to-many tables
    # and probably should still be, but this is easier to understand than dealing with Sqlalchemy cascaded
    # deletes on M-to-M secondary tables.

    @property
    def linked_partitions(self):
        return self._get_link_array('partitions', Partition, Partition.vid)

    def link_partition(self, p):
        return self._append_link('partitions', p.vid)

    def delink_partition(self, p):
        return self._remove_link('partitions', p.vid)

    @property
    def linked_tables(self):
        return self._get_link_array('tables', Table, Table.vid)

    def link_table(self, t):
        return self._append_link('tables', t.vid)

    def delink_table(self, t):
        return self._remove_link('tables', t.vid)

    @property
    def linked_manifests(self):
        return self._get_link_array('manifests', File, File.ref)

    def link_manifest(self, f):
        assert self.type_ != 'manifest'
        return self._append_link('manifests', f.ref)

    def delink_manifest(self, f):
        return self._remove_link('manifests', f.ref)

    @property
    def linked_stores(self):
        return self._get_link_array('stores', File, File.ref)

    def link_store(self, f):
        return self._append_link('stores', f.ref)

    def delink_store(self, f):
        return self._remove_link('stores', f.ref)

    @staticmethod
    def before_update(mapper, conn, target):
        """Set the column id number based on the table number and the sequence
        id for the column."""

        assert bool(target.ref), "File.ref can't be null (before_update)"

    @staticmethod
    def set_ref(target, value, oldvalue, initiator):
        "Strip non-numeric characters from a phone number"

        assert bool(value), "File.ref can't be null (set_ref)"

event.listen(File, 'before_insert', File.before_update)
event.listen(File, 'before_update', File.before_update)
event.listen(File.ref, 'set', File.set_ref)


# Day_28_02_chosun.py
import tensorflow as tf
import numpy as np
import re

# 텐서플로 2.0 프로그래밍
# # url = 'http://bit.ly/2Mc3SOV'
# url = 'https://raw.githubusercontent.com/greentec/greentec.github.io/master/public/other/data/chosundynasty/corpus.txt'
# file_path = tf.keras.utils.get_file(
#     'chosun.txt', url, cache_dir='.', cache_subdir='data')
# print(file_path)

def clean_str(string):
    string = re.sub(r"[^가-힣0-9]", " ", string)
    string = re.sub(r",", " , ", string)
    string = re.sub(r"!", " ! ", string)
    string = re.sub(r"\(", " \( ", string)
    string = re.sub(r"\)", " \) ", string)
    string = re.sub(r"\?", " \? ", string)
    string = re.sub(r"\s{2,}", " ", string)

    return string.strip()

def get_data():
    f = open('data/chosun.txt', 'r', encoding='utf-8')
    long_text = f.read()
    long_text = clean_str(long_text)
    long_text = long_text[:1000]
    f.close()

    tokens = long_text.split()
    vocab = ['UNK'] + sorted(set(tokens)) # unknown

    return tokens, vocab

def model_chosun_1():
    # 파일 읽기 -> 긴 문장 -> 토큰 분리 -> 인덱스로 변환 -> x, y 생성
    tokens, vocab = get_data()
    # print(len(tokens), len(vocab))        # 338 113

    word2idx = {w: i for i, w in enumerate(vocab)}
    # idx2word = {i: w for i, w in enumerate(vocab)}
    idx2word = np.array(vocab)

    # print(tokens[:5])                   # ['태조', '이성계', '선대의', '가계', '목조']
    # print(list(word2idx.items())[:3])   # [(',', 0), ('001', 1), ('002', 2)]
    # print(idx2word[:3])                 # [',' '001' '002']

    # tokens_idx = [vocab.index(w) for w in tokens] 밑에랑 같은코드 인덱스 위치 알려줌 (list)
    tokens_idx = [word2idx[w] for w in tokens]  # 위에랑 비교해서 훨씬 빠르다 (dict)라서
    # print(tokens_idx[:5])               # [104, 74, 47, 6, 34]

    # 25글자로 그 다음 글자 예측
    seq_len, vocab_size = 25, len(vocab)

    x = [tokens_idx[i:i+seq_len] for i in range(len(tokens) - seq_len)]
    y = [tokens_idx[i+seq_len] for i in range(len(tokens) - seq_len)]
    # print(x[0])     # [104, 74, 47, 6, 34, 79, 92, 45, 68, 8, 59, 93, 104, 7, 94, 12, 48, 56, 28, 4, 5, 67, 49, 77, 4]
    # print(y[:5])    # [5, 64, 0, 111, 4]

    # 모델 구축
    model = tf.keras.Sequential([
        tf.keras.layers.Input(shape=[seq_len]),
        tf.keras.layers.Embedding(vocab_size, 100),  # 12-2navermovie 참조 임베딩 필수!!(원핫으로 하면 메모리 상상초과) 입력(2차원), 출력(3차원)
        tf.keras.layers.LSTM(128, return_sequences=False),
        tf.keras.layers.Dense(vocab_size, activation='softmax'),
    ])

    model.compile(optimizer=tf.keras.optimizers.Adam(lr=0.01),
                  loss=tf.keras.losses.sparse_categorical_crossentropy,
                  metrics=['acc'])

    model.fit(x, y, epochs=10, batch_size=32, verbose=2)
    model.save('data/chosun_model_1.h5')

    # sent = '동헌에 나가 활을 쏘다'
    # act_like_writer_1(sent, model, word2idx, idx2word, seq_len)

def model_chosun_2():
    # 파일 읽기 -> 긴 문장 -> 토큰 분리 -> 인덱스로 변환 -> x, y 생성
    tokens, vocab = get_data()

    word2idx = {w: i for i, w in enumerate(vocab)}
    idx2word = np.array(vocab)

    tokens_idx = [word2idx[w] for w in tokens]  # 위에랑 비교해서 훨씬 빠르다 (dict)라서

    # 25글자로 그 다음 글자 예측
    seq_len, vocab_size = 25, len(vocab)

    # x = [tokens_idx[i:i+seq_len] for i in range(len(tokens) - seq_len)]
    # y = [tokens_idx[i+seq_len] for i in range(len(tokens) - seq_len)]

    # Dataset 생성 -> (seq_len+1) 분할 -> (x, y) 튜플 변환 -> 셔플 -> 배치 크기 분할 -> 원하는 만큼 무제한 반복
    sent_slices = tf.data.Dataset.from_tensor_slices(tokens_idx)
    # print(type(sent_slices))
    # print(sent_slices.take(2))

    # for takes in sent_slices.take(2):
    #     print(takes.numpy(), takes)
    # 195 tf.Tensor(195, shape=(), dtype=int32)
    # 150 tf.Tensor(150, shape=(), dtype=int32)

    sent_sequences = sent_slices.batch(seq_len+1, drop_remainder=True) # 자투리 버림
    # for takes in sent_sequences.take(2):
    #    print(takes.numpy())
    # [195 150 105  11  75 155 175 100 141 ...  107 153 131 208  47  48]
    # [156 167  47  31 146  35 208  47 150 ...  160 142 115 124  85 196]

    sent_xy = sent_sequences.map(lambda chunk: (chunk[:-1], chunk[-1])) # (x, y)튜플로 변환한 부분

    # for xx, yy in sent_xy.take(2):
    #     print(xx.numpy(), yy.numpy())
    # [195 150 105  11  75 155 175 ... 51 138 107 153 131 208  47] 48
    # [156 167  47  31 146  35 208 ... 91 208 160 142 115 124  85] 196

    steps_per_epoch = len(tokens_idx) // (seq_len+1)
    sent_shuffled = sent_xy.shuffle(buffer_size=steps_per_epoch)
    # for xx, yy in sent_shuffled.take(2):
    #     print(xx.numpy(), yy.numpy())
    # [156 167  47  31 146  35 208  ...  91 208 160 142 115 124  85] 196
    # [137  42 190 208  21 137  42  ...  208 184  73  42 185 208  32] 137

    # 문제
    # sent_batches에 들어있는 값을 출력하세요.
    batch_size = 32
    sent_batches = sent_shuffled.batch(batch_size)
    # for xx, yy in sent_batches.take(2):
    #    print(xx.shape, yy.shape)       # (12, 25) (12,)

    sent_infinite = sent_batches.repeat()

    # 모델 구축
    model = tf.keras.Sequential([
        tf.keras.layers.Input(shape=[seq_len]),
        tf.keras.layers.Embedding(vocab_size, 100),  # 12-2navermovie 참조 임베딩 필수!!(원핫으로 하면 메모리 상상초과) 입력(2차원), 출력(3차원)
        tf.keras.layers.LSTM(128, return_sequences=False),
        tf.keras.layers.Dense(vocab_size, activation='softmax'),
    ])

    model.compile(optimizer=tf.keras.optimizers.Adam(lr=0.01),
                  loss=tf.keras.losses.sparse_categorical_crossentropy,
                  metrics=['acc'])

    model.fit(sent_infinite, epochs=10, steps_per_epoch=steps_per_epoch, verbose=2)
    model.save('data/chosun_model_2.h5')

    sent = '동헌에 나가 활을 쏘다'
    # act_like_writer_1(sent, model, word2idx, idx2word, seq_len)

def act_like_writer_1(sent, model, word2idx, idx2word, seq_len):
    current = sent.split()

    for i in range(100):
        tokens = current[-seq_len:] # 맨밑 주석부분
        tokens_idx = [word2idx[w] if w in word2idx else word2idx['UNK'] for w in tokens]

        tokens_pad = tf.keras.preprocessing.sequence.pad_sequences( # pad_sequences는 2차원만 받는다
            [tokens_idx], maxlen=seq_len, value=word2idx['UNK']     # 그래서 2차원만들어줌
        )

        preds = model.predict(tokens_pad)
        # print(preds.shape)                # (1, 113)
        # preds_arg = np.argmax(preds, axis=1)
        preds_arg = np.argmax(preds[0])     # argmax를 통해 위치 찾음
        # print(preds_arg)                  # 4

        current.append(idx2word[preds_arg])

    print(current)

def act_like_writer_2(sent, model, word2idx, idx2word, seq_len):
    TOKEN_UNK = word2idx['UNK']
    current = [word2idx[w] if w in word2idx else TOKEN_UNK for w in sent.split()]

    filled = seq_len - len(current)
    if filled <= 0:
        filled = 0
    else:
        # current = current + [TOKEN_UNK] * filled      # difficult
        current = [TOKEN_UNK] * filled + current

    for i in range(100):
        tokens_idx = current[-seq_len:]
        # tokens_pad = [tokens_idx]             # 25번째에서 에러
        tokens_pad = np.array([tokens_idx])

        preds = model.predict(tokens_pad)
        preds_arg = np.argmax(preds[0])

        # print(i, preds_arg, idx2word[preds_arg])
        current.append(preds_arg)

    valid = current[filled + len(sent.split()):]
    print(sent, ' '.join(idx2word[valid]))

# 문제
# 모델을 읽어와서 전달받은 문장으로부터 새로운 문장을 생성하는 함수를 만드세요.
def load_model(sent, model_path):
    tokens, vocab = get_data()

    word2idx = {w: i for i, w in enumerate(vocab)}
    idx2word = np.array(vocab)

    model = tf.keras.models.load_model(model_path)
    # act_like_writer_1(sent, model, word2idx, idx2word, seq_len=25)
    act_like_writer_2(sent, model, word2idx, idx2word, seq_len=25)

# model_chosun_1()
model_chosun_2()

# load_model('동헌에 나가 활을 쏘다', 'data/chosun_model_1.h5')
# load_model('태조 이성계 아들 이방헌', 'data/chosun_model_1.h5')

load_model('태조 이성계 아들 이방헌', 'data/chosun_model_2.h5')

# [h e l l o] k라는 새로운 글자 얻음
# [e l l o k] t
# [l l o k t]


import requests
import datetime as dt

from requests.models import Response

USERNAME = "whzd"
TOKEN = "verysecretcodeomg"
PIXELA_URL = "https://pixe.la/v1/users/"
GRAPH_ID = "graph1"

# Create a user
user_parameters = {
    "token": "verysecretcodeomg",
    "username": "whzd",
    "agreeTermsOfService": "yes",
    "notMinor": "yes"
}

# response = requests.post(url=PIXELA_URL, json=user_parameters)
# print(response.text)

# Create a graph
graph_parameters = {
    "id": "graph1",
    "name": "Meditation Graph",
    "unit": "minutes",
    "type": "int",
    "color": "sora"
}

headers = {
    "X-USER-TOKEN": TOKEN
}

# response = requests.post(url=f"{PIXELA_URL}whzd/graphs", json=graph_parameters, headers=headers)
# print(response.text)

# Create a pixel
pixel_parameters = {
    # "date": str(dt.date.today()).replace("-", ""),
    "date": dt.datetime.now().strftime("%Y%m%d"),
    "quantity": "5"
}

# response = requests.post(url=f"{PIXELA_URL}{USERNAME}/graphs/{GRAPH_ID}", json=pixel_parameters, headers=headers)
# response.raise_for_status()
# print(response.json())

date = dt.datetime.now().strftime("%Y%m%d")

# Update a pixel
# response = requests.put(url=f"{PIXELA_URL}{USERNAME}/graphs/{GRAPH_ID}/{date}", json={"quantity": "10"}, headers=headers)
# print(response.json())

# Delete a pixel
response = requests.delete(url=f"{PIXELA_URL}{USERNAME}/graphs/{GRAPH_ID}/{date}", headers=headers)
print(response.json())


import torch
import torch.nn as nn
import torch.nn.functional as F

class QNetwork(nn.Module):
    def __init__(self, state_size, action_size, seed, fc1_units=64, fc2_units=64):
        super(QNetwork, self).__init__()
        self.action_size = action_size
        self.seed = torch.manual_seed(seed)
        self.fc1 = nn.Linear(state_size, fc1_units)
        self.fc2 = nn.Linear(fc1_units, fc2_units)
        self.adv = nn.Linear(fc2_units, action_size)
        self.val = nn.Linear(fc2_units, 1)

    def forward(self, state):
        x = F.relu(self.fc1(state))
        x = F.relu(self.fc2(x))
        adv = self.adv(x)
        val = self.val(x).expand(x.size(0), self.action_size)

        x = val + adv - adv.mean(1).unsqueeze(1).expand(x.size(0), self.action_size)
        return x