__docformat__ = 'restructuredtext en'

from collections import OrderedDict
import six
from six import string_types
from geoid.util import isimplify
from geoid.civick import GVid
from geoid import parse_to_gvid
from dateutil import parser
from sqlalchemy import event
from sqlalchemy import Column as SAColumn, Integer, UniqueConstraint
from sqlalchemy import String, ForeignKey
from sqlalchemy.orm import relationship, object_session, backref
from ambry.identity import ObjectNumber, PartialPartitionName, PartitionIdentity
from ambry.orm.columnstat import ColumnStat
from ambry.orm.dataset import Dataset
from ambry.util import Constant
import logging
from ambry.util import get_logger

logger = get_logger(__name__)
# logger.setLevel(logging.DEBUG)

from . import Base, MutationDict, MutationList, JSONEncodedObj, BigIntegerType

class PartitionDisplay(object):
    """Helper object to select what to display for titles and descriptions"""

    def __init__(self, p):

        self._p = p

        desc_used = False
        self.title = self._p.title
        self.description = ''

        if not self.title:
            self.title = self._p.table.description
            desc_used = True

        if not self.title:
            self.title = self._p.vname

        if not desc_used:
            self.description = self._p.description.strip('.') + '.' if self._p.description else ''

        self.notes = self._p.notes

    @property
    def geo_description(self):
        """Return a description of the geographic extents, using the largest scale
        space and grain coverages"""

        sc = self._p.space_coverage
        gc = self._p.grain_coverage

        if sc and gc:
            if parse_to_gvid(gc[0]).level == 'state' and parse_to_gvid(sc[0]).level == 'state':
                return parse_to_gvid(sc[0]).geo_name
            else:
                return ("{} in {}".format(
                    parse_to_gvid(gc[0]).level_plural.title(),
                    parse_to_gvid(sc[0]).geo_name))
        elif sc:
            return parse_to_gvid(sc[0]).geo_name.title()
        elif sc:
            return parse_to_gvid(gc[0]).level_plural.title()
        else:
            return ''

    @property
    def time_description(self):
        """String description of the year or year range"""

        tc = [t for t in self._p.time_coverage if t]

        if not tc:
            return ''

        mn = min(tc)
        mx = max(tc)

        if not mn and not mx:
            return ''
        elif mn == mx:
            return mn
        else:
            return "{} to {}".format(mn, mx)

    @property
    def sub_description(self):
        """Time and space dscription"""
        gd = self.geo_description
        td = self.time_description

        if gd and td:
            return '{}, {}. {} Rows.'.format(gd, td, self._p.count)
        elif gd:
            return '{}. {} Rows.'.format(gd, self._p.count)
        elif td:
            return '{}. {} Rows.'.format(td, self._p.count)
        else:
            return '{} Rows.'.format(self._p.count)

class Partition(Base):
    __tablename__ = 'partitions'

    STATES = Constant()
    STATES.SYNCED = 'synced'
    STATES.CLEANING = 'cleaning'
    STATES.CLEANED = 'cleaned'
    STATES.PREPARING = 'preparing'
    STATES.PREPARED = 'prepared'
    STATES.BUILDING = 'building'
    STATES.BUILT = 'built'
    STATES.COALESCING = 'coalescing'
    STATES.COALESCED = 'coalesced'
    STATES.ERROR = 'error'
    STATES.FINALIZING = 'finalizing'
    STATES.FINALIZED = 'finalized'
    STATES.INSTALLING = 'installing'
    STATES.INSTALLED = 'installed'

    TYPE = Constant
    TYPE.SEGMENT = 's'
    TYPE.UNION = 'u'

    sequence_id = SAColumn('p_sequence_id', Integer)
    vid = SAColumn('p_vid', String(16), primary_key=True, nullable=False)
    id = SAColumn('p_id', String(13), nullable=False)
    d_vid = SAColumn('p_d_vid', String(13), ForeignKey('datasets.d_vid'), nullable=False, index=True)
    t_vid = SAColumn('p_t_vid', String(15), ForeignKey('tables.t_vid'), nullable=False, index=True)
    name = SAColumn('p_name', String(200), nullable=False, index=True)
    vname = SAColumn('p_vname', String(200), unique=True, nullable=False, index=True)
    fqname = SAColumn('p_fqname', String(200), unique=True, nullable=False, index=True)

    title = SAColumn('p_title', String())
    description = SAColumn('p_description', String())
    notes = SAColumn('p_notes', String())

    cache_key = SAColumn('p_cache_key', String(200), unique=True, nullable=False, index=True)
    parent_vid = SAColumn('p_p_vid', String(16), ForeignKey('partitions.p_vid'), nullable=True, index=True)
    ref = SAColumn('p_ref', String(16), index=True,
                   doc='VID reference to an eariler version to use instead of this one.')
    type = SAColumn('p_type', String(20), default=TYPE.UNION,
                    doc='u - normal partition, s - segment')
    table_name = SAColumn('p_table_name', String(50))
    time = SAColumn('p_time', String(20))  # FIXME: add helptext
    space = SAColumn('p_space', String(50))
    grain = SAColumn('p_grain', String(50))
    variant = SAColumn('p_variant', String(50))
    format = SAColumn('p_format', String(50))
    segment = SAColumn('p_segment', Integer,
                       doc='Part of a larger partition. segment_id is usually also a source ds_id')
    epsg = SAColumn('p_epsg', Integer, doc='EPSG SRID for the reference system of a geographic dataset. ')

    # The partition could hold data that is considered a dimension -- if multiple datasets
    # were joined, that dimension would be a dimension column, but it only has a single
    # value in each partition.
    # That could be part of the name, or it could be declared in a table, with a single value for all of the
    # rows in a partition.

    min_id = SAColumn('p_min_id', BigIntegerType)
    max_id = SAColumn('p_max_id', BigIntegerType)
    count = SAColumn('p_count', Integer)
    state = SAColumn('p_state', String(50))
    data = SAColumn('p_data', MutationDict.as_mutable(JSONEncodedObj))

    space_coverage = SAColumn('p_scov', MutationList.as_mutable(JSONEncodedObj))
    time_coverage = SAColumn('p_tcov', MutationList.as_mutable(JSONEncodedObj))
    grain_coverage = SAColumn('p_gcov', MutationList.as_mutable(JSONEncodedObj))

    installed = SAColumn('p_installed', String(100))
    _location = SAColumn('p_location', String(100))  # Location of the data file

    __table_args__ = (
        # ForeignKeyConstraint( [d_vid, d_location], ['datasets.d_vid','datasets.d_location']),
        UniqueConstraint('p_sequence_id', 'p_d_vid', name='_uc_partitions_1'),
    )

    # For the primary table for the partition. There is one per partition, but a table
    # can be primary in multiple partitions.
    table = relationship('Table', backref='partitions', foreign_keys='Partition.t_vid')

    stats = relationship(ColumnStat, backref='partition', cascade='all, delete, delete-orphan')

    children = relationship('Partition', backref=backref('parent', remote_side=[vid]), cascade='all')

    _bundle = None  # Set when returned from a bundle.
    _datafile = None  # TODO: Unused variable.
    _datafile_writer = None  # TODO: Unused variable.
    _stats_dict = None

    @property
    def identity(self):
        """Return this partition information as a PartitionId."""

        if self.dataset is None:
            # The relationship will be null until the object is committed
            s = object_session(self)

            ds = s.query(Dataset).filter(Dataset.id_ == self.d_id).one()
        else:
            ds = self.dataset

        d = {
            'id': self.id,
            'vid': self.vid,
            'name': self.name,
            'vname': self.vname,
            'ref': self.ref,
            'space': self.space,
            'time': self.time,
            'table': self.table_name,
            'grain': self.grain,
            'variant': self.variant,
            'segment': self.segment,
            'format': self.format if self.format else 'db'
        }

        return PartitionIdentity.from_dict(dict(list(ds.dict.items()) + list(d.items())))

    @property
    def display(self):
        """Return an acessor object to get display titles and descriptions"""
        return PartitionDisplay(self)

    @property
    def bundle(self):
        return self._bundle  # Set externally, such as Bundle.wrap_partition

    @property
    def is_segment(self):
        return self.type == self.TYPE.SEGMENT

    @property
    def headers(self):
        return [c.name for c in self.table.columns]

    def __repr__(self):
        return '<partition: {} {}>'.format(self.vid, self.vname)

    def set_stats(self, stats):

        self.stats[:] = []  # Delete existing stats

        for c in self.table.columns:

            if c.name not in stats:
                continue

            d = stats[c.name].dict

            del d['name']
            del d['flags']
            cs = ColumnStat(p_vid=self.vid, d_vid=self.d_vid, c_vid=c.vid, **d)
            self.stats.append(cs)

    def parse_gvid_or_place(self, gvid_or_place):
        try:
            return parse_to_gvid(gvid_or_place)
        except KeyError:

            places = list(self._bundle._library.search.search_identifiers(gvid_or_place))

            if not places:
                err_msg = "Failed to find space identifier '{}' in full " \
                          "text identifier search  for partition '{}'" \
                    .format(gvid_or_place, str(self.identity))
                self._bundle.error(err_msg)
                return None

            return parse_to_gvid(places[0].vid)

    def set_coverage(self, stats):
        """"Extract time space and grain coverage from the stats and store them in the partition"""
        from ambry.util.datestimes import expand_to_years

        scov = set()
        tcov = set()
        grains = set()

        def summarize_maybe(gvid):
            try:
                return parse_to_gvid(gvid).summarize()
            except:
                return None

        def simplifiy_maybe(values, column):

            parsed = []

            for gvid in values:
                # The gvid should not be a st
                if gvid is None or gvid == 'None':
                    continue
                try:
                    parsed.append(parse_to_gvid(gvid))
                except ValueError as e:
                    if self._bundle:
                        self._bundle.warn("While analyzing geo coverage in final partition stage, " +
                                           "Failed to parse gvid '{}' in {}.{}: {}"
                                           .format(str(gvid), column.table.name, column.name, e))

            try:
                return isimplify(parsed)
            except:
                return None

        def int_maybe(year):
            try:
                return int(year)
            except:
                return None

        for c in self.table.columns:

            if c.name not in stats:
                continue

            try:
                if stats[c.name].is_gvid or stats[c.name].is_geoid:
                    scov |= set(x for x in simplifiy_maybe(stats[c.name].uniques, c))
                    grains |= set(summarize_maybe(gvid) for gvid in stats[c.name].uniques)

                elif stats[c.name].is_year:
                    tcov |= set(int_maybe(x) for x in stats[c.name].uniques)

                elif stats[c.name].is_date:
                    # The fuzzy=True argument allows ignoring the '-' char in dates produced by .isoformat()
                    try:
                        tcov |= set(parser.parse(x, fuzzy=True).year if isinstance(x, string_types) else x.year for x in
                                    stats[c.name].uniques)
                    except ValueError:
                        pass

            except Exception as e:
                self._bundle.error("Failed to set coverage for column '{}', partition '{}': {}"
                                   .format(c.name, self.identity.vname, e))
                raise

        # Space Coverage

        if 'source_data' in self.data:

            for source_name, source in list(self.data['source_data'].items()):
                scov.add(self.parse_gvid_or_place(source['space']))

        if self.identity.space:  # And from the partition name
            try:
                scov.add(self.parse_gvid_or_place(self.identity.space))
            except ValueError:
                # Couldn't parse the space as a GVid
                pass

        # For geo_coverage, only includes the higher level summary levels, counties, states,
        # places and urban areas.
        self.space_coverage = sorted([str(x) for x in scov if bool(x) and x.sl
                                      in (10, 40, 50, 60, 160, 400)])

        #
        # Time Coverage

        # From the source
        # If there was a time value in the source that this partition was created from, then
        # add it to the years.
        if 'source_data' in self.data:
            for source_name, source in list(self.data['source_data'].items()):
                if 'time' in source:
                    for year in expand_to_years(source['time']):
                        if year:
                            tcov.add(year)

        # From the partition name
        if self.identity.name.time:
            for year in expand_to_years(self.identity.name.time):
                if year:
                    tcov.add(year)

        self.time_coverage = [t for t in tcov if t]

        #
        # Grains

        if 'source_data' in self.data:
            for source_name, source in list(self.data['source_data'].items()):
                if 'grain' in source:
                    grains.add(source['grain'])

        self.grain_coverage = sorted(str(g) for g in grains if g)

    @property
    def dict(self):
        """A dict that holds key/values for all of the properties in the
        object.

        :return:

        """

        d = {p.key: getattr(self, p.key) for p in self.__mapper__.attrs
             if p.key not in ('table', 'dataset', '_codes', 'stats', 'data', 'process_records')}

        if self.data:
            # Copy data fields into top level dict, but don't overwrite existind values.
            for k, v in six.iteritems(self.data):
                if k not in d and k not in ('table', 'stats', '_codes', 'data'):
                    d[k] = v

        return d

    @property
    def detail_dict(self):
        """A more detailed dict that includes the descriptions, sub descriptions, table
        and columns."""

        d = self.dict

        def aug_col(c):
            d = c.dict
            d['stats'] = [s.dict for s in c.stats]
            return d

        d['table'] = self.table.dict
        d['table']['columns'] = [aug_col(c) for c in self.table.columns]

        return d

    @property
    def stats_dict(self):

        class Bunch(object):
            """Dict and object access to properties"""

            def __init__(self, o):
                self.__dict__.update(o)

            def __str__(self):
                return str(self.__dict__)

            def __repr__(self):
                return repr(self.__dict__)

            def keys(self):
                return list(self.__dict__.keys())

            def items(self):
                return list(self.__dict__.items())

            def iteritems(self):
                return iter(self.__dict__.items())

            def __getitem__(self, k):
                if k in self.__dict__:
                    return self.__dict__[k]
                else:
                    from . import ColumnStat
                    return ColumnStat(hist=[])

        if not self._stats_dict:
            cols = {s.column.name: Bunch(s.dict) for s in self.stats}

            self._stats_dict = Bunch(cols)

        return self._stats_dict

    def build_sample(self):

        name = self.table.name

        count = int(
            self.database.connection.execute('SELECT count(*) FROM "{}"'.format(name)).fetchone()[0])

        skip = count / 20

        if count > 100:
            sql = 'SELECT * FROM "{}" WHERE id % {} = 0 LIMIT 20'.format(name, skip)
        else:
            sql = 'SELECT * FROM "{}" LIMIT 20'.format(name)

        sample = []

        for j, row in enumerate(self.database.connection.execute(sql)):
            sample.append(list(row.values()))

        self.record.data['sample'] = sample

        s = self.bundle.database.session
        s.merge(self.record)
        s.commit()

    @property
    def row(self):
        # Use an Ordered Dict to make it friendly to creating CSV files.
        SKIP_KEYS = [
            'sequence_id', 'vid', 'id', 'd_vid', 't_vid', 'min_key', 'max_key',
            'installed', 'ref', 'count', 'state', 'data', 'space_coverage',
            'time_coverage', 'grain_coverage', 'name', 'vname', 'fqname', 'cache_key'
        ]

        d = OrderedDict([('table', self.table.name)] +
                        [(p.key, getattr(self, p.key)) for p in self.__mapper__.attrs
                         if p.key not in SKIP_KEYS])
        return d

    def update(self, **kwargs):

        if 'table' in kwargs:
            del kwargs['table']  # In source_schema.csv, this is the name of the table, not the object

        for k, v in list(kwargs.items()):
            if hasattr(self, k):
                setattr(self, k, v)

    def finalize(self, ps=None):

        self.state = self.STATES.FINALIZING

        # Write the stats for this partition back into the partition

        with self.datafile.writer as w:
            for i, c in enumerate(self.table.columns, 1):
                wc = w.column(i)
                assert wc.pos == c.sequence_id, (c.name, wc.pos, c.sequence_id)
                wc.name = c.name
                wc.description = c.description
                wc.type = c.python_type.__name__
                self.count = w.n_rows
            w.finalize()

        if self.type == self.TYPE.UNION:
            ps.update('Running stats ', state='running')
            stats = self.datafile.run_stats()

            self.set_stats(stats)
            self.set_coverage(stats)

        self._location = 'build'

        self.title = PartitionDisplay(self).title
        self.description = PartitionDisplay(self).description

        self.state = self.STATES.FINALIZED

    # =============
    # These methods are a bit non-cohesive, since they require the _bundle value to be set, which is
    # set externally, when the object is retured from a bundle.

    def clean(self):
        """Remove all built files and return the partition to a newly-created state"""
        if self.datafile:
            self.datafile.remove()

    @property
    def location(self):

        base_location = self._location

        if not base_location:
            return None

        if self._bundle.build_fs.exists(base_location):
            if self._bundle.build_fs.hashsyspath(base_location):
                return self._bundle.build_fs.getsyspath(base_location)

        return base_location

    @location.setter
    def location(self, v):
        self._location = v

    @property
    def datafile(self):
        from ambry.exc import NotFoundError

        if self.is_local:
            # Use the local version, if it exists
            logger.debug('datafile: Using local datafile {}'.format(self.vname))
            return self.local_datafile
        else:
            # If it doesn't try to get the remote.
            try:
                logger.debug('datafile: Using remote datafile {}'.format(self.vname))
                return self.remote_datafile
            except NotFoundError:
                # If the remote doesnt exist, return the local, so the caller can call  exists() on it,
                # get its path, etc.
                return self.local_datafile

    @property
    def local_datafile(self):
        """Return the datafile for this partition, from the build directory, the remote, or the warehouse"""
        from ambry_sources import MPRowsFile
        from fs.errors import ResourceNotFoundError
        from ambry.orm.exc import NotFoundError

        try:
            return MPRowsFile(self._bundle.build_fs, self.cache_key)

        except ResourceNotFoundError:
            raise NotFoundError(
                'Could not locate data file for partition {} (local)'.format(self.identity.fqname))

    @property
    def remote(self):
        """
        Return the remote for this partition

        :return:

        """
        from ambry.exc import NotFoundError

        ds = self.dataset

        if 'remote_name' not in ds.data:
            raise NotFoundError('Could not determine remote for partition: {}'.format(self.identity.fqname))

        return self._bundle.library.remote(ds.data['remote_name'])

    @property
    def remote_datafile(self):

        from fs.errors import ResourceNotFoundError
        from ambry.exc import AccessError, NotFoundError
        from boto.exception import S3ResponseError

        try:

            from ambry_sources import MPRowsFile

            remote = self.remote

            datafile = MPRowsFile(remote.fs, self.cache_key)

            if not datafile.exists:
                raise NotFoundError(
                    'Could not locate data file for partition {} from remote {} : file does not exist'
                        .format(self.identity.fqname, remote))

        except ResourceNotFoundError as e:
            raise NotFoundError('Could not locate data file for partition {} (remote): {}'
                                .format(self.identity.fqname, e))
        except S3ResponseError as e:
            # HACK. It looks like we get the response error with an access problem when
            # we have access to S3, but the file doesn't exist.
            raise NotFoundError("Can't access MPR file for {} in remote {}".format(self.cache_key, remote.fs))

        return datafile

    @property
    def is_local(self):
        """Return true is the partition file is local"""
        from ambry.orm.exc import NotFoundError
        try:
            if self.local_datafile.exists:
                return True
        except NotFoundError:
            pass

        return False

    def localize(self, ps=None):
        """Copy a non-local partition file to the local build directory"""
        from filelock import FileLock
        from ambry.util import ensure_dir_exists
        from ambry_sources import MPRowsFile
        from fs.errors import ResourceNotFoundError

        if self.is_local:
            return

        local = self._bundle.build_fs

        b = self._bundle.library.bundle(self.identity.as_dataset().vid)

        remote = self._bundle.library.remote(b)

        lock_path = local.getsyspath(self.cache_key + '.lock')

        ensure_dir_exists(lock_path)

        lock = FileLock(lock_path)

        if ps:
            ps.add_update(message='Localizing {}'.format(self.identity.name),
                          partition=self,
                          item_type='bytes',
                          state='downloading')

        if ps:
            def progress(bts):
                if ps.rec.item_total is None:
                    ps.rec.item_count = 0

                if not ps.rec.data:
                    ps.rec.data = {}  # Should not need to do this.
                    return self

                item_count = ps.rec.item_count + bts
                ps.rec.data['updates'] = ps.rec.data.get('updates', 0) + 1

                if ps.rec.data['updates'] % 32 == 1:
                    ps.update(message='Localizing {}'.format(self.identity.name),
                              item_count=item_count)
        else:
            from ambry.bundle.process import call_interval
            @call_interval(5)
            def progress(bts):
                self._bundle.log("Localizing {}. {} bytes downloaded".format(self.vname, bts))

        def exception_cb(e):
            raise e

        with lock:
            # FIXME! This won't work with remote ( http) API, only FS ( s3:, file:)

            if self.is_local:
                return self

            try:
                with remote.fs.open(self.cache_key + MPRowsFile.EXTENSION, 'rb') as f:
                    event = local.setcontents_async(self.cache_key + MPRowsFile.EXTENSION,
                                                    f,
                                                    progress_callback=progress,
                                                    error_callback=exception_cb)
                    event.wait()
                    if ps:
                        ps.update_done()
            except ResourceNotFoundError as e:
                from ambry.orm.exc import NotFoundError
                raise NotFoundError("Failed to get MPRfile '{}' from {}: {} "
                                    .format(self.cache_key, remote.fs, e))

        return self

    @property
    def reader(self):
        from ambry.orm.exc import NotFoundError
        from fs.errors import ResourceNotFoundError
        """The reader for the datafile"""

        try:
            return self.datafile.reader
        except ResourceNotFoundError:
            raise NotFoundError("Failed to find partition file, '{}' "
                                .format(self.datafile.path))

    def select(self, predicate=None, headers=None):
        """
        Select rows from the reader using a predicate to select rows and and itemgetter to return a
        subset of elements
        :param predicate: If defined, a callable that is called for each row, and if it returns true, the
        row is included in the output.
        :param headers: If defined, a list or tuple of header names to return from each row
        :return: iterable of results

        WARNING: This routine works from the reader iterator, which returns RowProxy objects. RowProxy objects
        are reused, so if you construct a list directly from the output from this method, the list will have
        multiple copies of a single RowProxy, which will have as an inner row the last result row. If you will
        be directly constructing a list, use a getter that extracts the inner row, or which converts the RowProxy
        to a dict:

            list(s.datafile.select(lambda r: r.stusab == 'CA', lambda r: r.dict ))

        """

        # FIXME; in Python 3, use yield from
        with self.reader as r:
            for row in r.select(predicate, headers):
                yield row

    def __iter__(self):
        """ Iterator over the partition, returning RowProxy objects.
        :return: a generator
        """

        with self.reader as r:
            for row in r:
                yield row

    @property
    def analysis(self):
        """Return an AnalysisPartition proxy, which wraps this partition to provide acess to
        dataframes, shapely shapes and other analysis services"""
        if isinstance(self, PartitionProxy):
            return AnalysisPartition(self._obj)
        else:
            return AnalysisPartition(self)

    @property
    def measuredim(self):
        """Return a MeasureDimension proxy, which wraps the partition to provide access to
        columns in terms of measures and dimensions"""

        if isinstance(self, PartitionProxy):
            return MeasureDimensionPartition(self._obj)
        else:
            return MeasureDimensionPartition(self)

    # ============================

    def update_id(self, sequence_id=None):
        """Alter the sequence id, and all of the names and ids derived from it. This
        often needs to be done after an IntegrityError in a multiprocessing run"""

        if sequence_id:
            self.sequence_id = sequence_id

        self._set_ids(force=True)

        if self.dataset:
            self._update_names()

    def _set_ids(self, force=False):

        if not self.sequence_id:
            from .exc import DatabaseError

            raise DatabaseError('Sequence ID must be set before insertion')

        if not self.vid or force:
            assert bool(self.d_vid)
            assert bool(self.sequence_id)
            don = ObjectNumber.parse(self.d_vid)
            assert don.revision
            on = don.as_partition(self.sequence_id)
            self.vid = str(on.rev(don.revision))
            self.id = str(on.rev(None))

        if not self.data:
            self.data = {}

    def _update_names(self):
        """Update the derived names"""

        d = dict(
            table=self.table_name,
            time=self.time,
            space=self.space,
            grain=self.grain,
            variant=self.variant,
            segment=self.segment
        )

        assert self.dataset

        name = PartialPartitionName(**d).promote(self.dataset.identity.name)

        self.name = str(name.name)
        self.vname = str(name.vname)
        self.cache_key = name.cache_key
        self.fqname = str(self.identity.fqname)

    @staticmethod
    def before_insert(mapper, conn, target):
        """event.listen method for Sqlalchemy to set the sequence for this
        object and create an ObjectNumber value for the id_"""

        target._set_ids()

        if target.name and target.vname and target.cache_key and target.fqname and not target.dataset:
            return

        Partition.before_update(mapper, conn, target)

    @staticmethod
    def before_update(mapper, conn, target):
        target._update_names()

    @staticmethod
    def before_delete(mapper, conn, target):
        pass

event.listen(Partition, 'before_insert', Partition.before_insert)
event.listen(Partition, 'before_update', Partition.before_update)
event.listen(Partition, 'before_delete', Partition.before_delete)

class PartitionProxy(object):
    __slots__ = ["_obj", "__weakref__"]

    def __init__(self, obj):
        object.__setattr__(self, "_obj", obj)

    #
    # proxying (special cases)
    #
    def __getattr__(self, name):
        return getattr(object.__getattribute__(self, "_obj"), name)

    def __delattr__(self, name):
        delattr(object.__getattribute__(self, "_obj"), name)

    def __setattr__(self, name, value):
        setattr(object.__getattribute__(self, "_obj"), name, value)

    def __nonzero__(self):
        return bool(object.__getattribute__(self, "_obj"))

    def __str__(self):
        return "<{}: {}>".format(type(self), str(object.__getattribute__(self, "_obj")))

    def __repr__(self):
        return "<{}: {}>".format(type(self), repr(object.__getattribute__(self, "_obj")))

    def __iter__(self):
        return iter(object.__getattribute__(self, "_obj"))

class AnalysisPartition(PartitionProxy):
    """A subclass of Partition with methods designed for analysis with Pandas. It is produced from
    the partitions analysis property"""

    def dataframe(self, predicate=None, filtered_columns=None, columns=None, df_class=None):
        """Return the partition as a Pandas dataframe

        :param predicate: If defined, a callable that is called for each row, and if it returns true, the
        row is included in the output.
        :param filtered_columns: If defined, the value is a dict of column names and
        associated values. Only rows where all of the named columms have the given values will be returned.
        Setting the argument will overwrite any value set for the predicate
        :param columns: A list or tuple of column names to return

        :return: Pandas dataframe

        """

        from operator import itemgetter
        from ambry.pands import AmbryDataFrame

        df_class = df_class or AmbryDataFrame

        if columns:
            ig = itemgetter(*columns)
        else:
            ig = None
            columns = self.table.header

        if filtered_columns:

            def maybe_quote(v):
                from six import string_types
                if isinstance(v, string_types):
                    return '"{}"'.format(v)
                else:
                    return v

            code = ' and '.join("row.{} == {}".format(k, maybe_quote(v))
                                for k, v in filtered_columns.items())

            predicate = eval('lambda row: {}'.format(code))

        if predicate:
            def yielder():
                for row in self.reader:
                    if predicate(row):
                        if ig:
                            yield ig(row)
                        else:
                            yield row.dict

            df = df_class(yielder(), columns=columns, partition=self.measuredim)

            return df

        else:

            def yielder():
                for row in self.reader:
                    yield row.values()

            # Put column names in header order
            columns = [c for c in self.table.header if c in columns]

            return df_class(yielder(), columns=columns, partition=self.measuredim)

    def geoframe(self, simplify=None, predicate=None, crs=None, epsg=None):
        """
        Return geopandas dataframe

        :param simplify: Integer or None. Simplify the geometry to a tolerance, in the units of the geometry.
        :param predicate: A single-argument function to select which records to include in the output.
        :param crs: Coordinate reference system information
        :param epsg: Specifiy the CRS as an EPGS number.
        :return: A Geopandas GeoDataFrame
        """
        import geopandas
        from shapely.wkt import loads
        from fiona.crs import from_epsg

        if crs is None and epsg is None and self.epsg is not None:
            epsg = self.epsg

        if crs is None:
            try:
                crs = from_epsg(epsg)
            except TypeError:
                raise TypeError('Must set either crs or epsg for output.')

        df = self.dataframe(predicate=predicate)
        geometry = df['geometry']

        if simplify:
            s = geometry.apply(lambda x: loads(x).simplify(simplify))
        else:
            s = geometry.apply(lambda x: loads(x))

        df['geometry'] = geopandas.GeoSeries(s)

        return geopandas.GeoDataFrame(df, crs=crs, geometry='geometry')

    def shapes(self, simplify=None, predicate=None):
        """
        Return geodata as a list of Shapely shapes

        :param simplify: Integer or None. Simplify the geometry to a tolerance, in the units of the geometry.
        :param predicate: A single-argument function to select which records to include in the output.

        :return: A list of Shapely objects
        """

        from shapely.wkt import loads

        if not predicate:
            predicate = lambda row: True

        if simplify:
            return [loads(row.geometry).simplify(simplify) for row in self if predicate(row)]
        else:
            return [loads(row.geometry) for row in self if predicate(row)]

    def patches(self, basemap, simplify=None, predicate=None, args_f=None, **kwargs):
        """
        Return geodata as a list of Matplotlib patches

        :param basemap: A mpl_toolkits.basemap.Basemap
        :param simplify: Integer or None. Simplify the geometry to a tolerance, in the units of the geometry.
        :param predicate: A single-argument function to select which records to include in the output.
        :param args_f: A function that takes a row and returns a dict of additional args for the Patch constructor

        :param kwargs: Additional args to be passed to the descartes Path constructor
        :return: A list of patch objects
        """
        from descartes import PolygonPatch
        from shapely.wkt import loads
        from shapely.ops import transform

        if not predicate:
            predicate = lambda row: True

        def map_xform(x, y, z=None):
            return basemap(x, y)

        def make_patch(shape, row):

            args = dict(kwargs.items())

            if args_f:
                args.update(args_f(row))

            return PolygonPatch(transform(map_xform, shape), **args)

        def yield_patches(row):

            if simplify:
                shape = loads(row.geometry).simplify(simplify)
            else:
                shape = loads(row.geometry)

            if shape.geom_type == 'MultiPolygon':
                for subshape in shape.geoms:
                    yield make_patch(subshape, row)
            else:
                yield make_patch(shape, row)

        return [patch for row in self if predicate(row)
                for patch in yield_patches(row)]

class MeasureDimensionPartition(PartitionProxy):
    """A partition proxy for accessing measure and dimensions. When returning a column, it returns
    a PartitionColumn, which proxies the table column while adding partition specific functions. """

    def __init__(self, obj):

        super(MeasureDimensionPartition, self).__init__(obj)

        self.filters = {}

    def column(self, c_name):

        return PartitionColumn(self.table.column(c_name), self)

    @property
    def columns(self):
        """Iterate over all columns"""

        return [PartitionColumn(c, self) for c in self.table.columns]

    @property
    def primary_columns(self):
        """Iterate over the primary columns, columns which do not have a parent"""

        return [c for c in self.columns if not c.parent]

    @property
    def dimensions(self):
        """Iterate over all dimensions"""
        from ambry.valuetype.core import ROLE

        return [c for c in self.columns if c.role == ROLE.DIMENSION]

    @property
    def primary_dimensions(self):
        """Iterate over the primary columns, columns which do not have a parent and have a
        cardinality greater than 1"""
        from ambry.valuetype.core import ROLE

        return [c for c in self.columns
                if not c.parent and c.role == ROLE.DIMENSION and c.pstats.nuniques > 1]

    @property
    def measures(self):
        """Iterate over all measures"""
        from ambry.valuetype.core import ROLE

        return [c for c in self.columns if c.role == ROLE.MEASURE]

    def measure(self, vid):
        """Return a measure, given its vid or another reference"""

        from ambry.orm import Column

        if isinstance(vid, PartitionColumn):
            return vid
        elif isinstance(vid, Column):
            return PartitionColumn(vid)
        else:
            return PartitionColumn(self.table.column(vid), self)

    def dimension(self, vid):
        """Return a dimention, given its vid or another reference"""
        from ambry.orm import Column

        if isinstance(vid, PartitionColumn):
            return vid
        elif isinstance(vid, Column):
            return PartitionColumn(vid)
        else:
            return PartitionColumn(self.table.column(vid), self)

    @property
    def primary_measures(self):
        """Iterate over the primary measures, columns which do not have a parent"""

        return [c for c in self.measures if not c.parent]

    @property
    def dict(self):

        d = self.detail_dict

        d['dimension_sets'] = self.enumerate_dimension_sets()

        return d

    def dataframe(self, measure, p_dim, s_dim=None, filters={}, df_class=None):
        """
        Return a dataframe with a sumse of the columns of the partition, including a measure and one
        or two dimensions. FOr dimensions that have labels, the labels are included

        The returned dataframe will have extra properties to describe the conversion:

        * plot_axes: List of dimension names for the first and second axis
        * labels: THe names of the label columns for the axes
        * filtered: The `filters` dict
        * floating: The names of primary dimensions that are not axes nor filtered

        THere is also an iterator, `rows`, which returns the header and then all of the rows.

        :param measure: The column names of one or more measures
        :param p_dim: The primary dimension. This will be the index of the dataframe.
        :param s_dim: a secondary dimension. The returned frame will be unstacked on this dimension
        :param filters: A dict of column names, mapped to a column value, indicating rows to select. a
        row that passes the filter must have the values for all given rows; the entries are ANDED
        :param df_class:
        :return: a Dataframe, with extra properties
        """

        import numpy as np

        measure = self.measure(measure)

        p_dim = self.dimension(p_dim)

        assert p_dim

        if s_dim:
            s_dim = self.dimension(s_dim)

        columns = set([measure.name, p_dim.name])

        if p_dim.label:

            # For geographic datasets, also need the gvid
            if p_dim.geoid:
                columns.add(p_dim.geoid.name)

            columns.add(p_dim.label.name)

        if s_dim:

            columns.add(s_dim.name)

            if s_dim.label:
                columns.add(s_dim.label.name)

        def maybe_quote(v):
            from six import string_types
            if isinstance(v, string_types):
                return '"{}"'.format(v)
            else:
                return v

        # Create the predicate to filter out the filtered dimensions
        if filters:

            selected_filters = []

            for k, v in filters.items():
                if isinstance(v, dict):
                    # The filter is actually the whole set of possible options, so
                    # just select the first one
                    v = v.keys()[0]

                selected_filters.append("row.{} == {}".format(k, maybe_quote(v)))

            code = ' and '.join(selected_filters)

            predicate = eval('lambda row: {}'.format(code))
        else:
            code = None

            def predicate(row):
                return True

        df = self.analysis.dataframe(predicate, columns=columns, df_class=df_class)

        if df is None or df.empty or len(df) == 0:
            return None

        # So we can track how many records were aggregated into each output row
        df['_count'] = 1

        def aggregate_string(x):
            return ', '.join(set(str(e) for e in x))

        agg = {
            '_count': 'count',

        }

        for col_name in columns:
            c = self.column(col_name)

            # The primary and secondary dimensions are put into the index by groupby
            if c.name == p_dim.name or (s_dim and c.name == s_dim.name):
                continue

            # FIXME! This will only work if the child is only level from the parent. Should
            # have an acessor for the top level.
            if c.parent and (c.parent == p_dim.name or (s_dim and c.parent == s_dim.name)):
                continue

            if c.is_measure:
                agg[c.name] = np.mean

            if c.is_dimension:
                agg[c.name] = aggregate_string

        plot_axes = [p_dim.name]

        if s_dim:
            plot_axes.append(s_dim.name)

        df = df.groupby(list(columns - set([measure.name]))).agg(agg).reset_index()

        df._metadata = ['plot_axes', 'filtered', 'floating', 'labels', 'dimension_set', 'measure']

        df.plot_axes = [c for c in plot_axes]
        df.filtered = filters

        # Dimensions that are not specified as axes nor filtered
        df.floating = list(set(c.name for c in self.primary_dimensions) -
                           set(df.filtered.keys()) -
                           set(df.plot_axes))

        df.labels = [self.column(c).label.name if self.column(c).label else c for c in df.plot_axes]

        df.dimension_set = self.dimension_set(p_dim, s_dim=s_dim)

        df.measure = measure.name

        def rows(self):
            yield ['id'] + list(df.columns)

            for t in df.itertuples():
                yield list(t)

        # Really should not do this, but I don't want to re-build the dataframe with another
        # class
        df.__class__.rows = property(rows)

        return df

    def dimension_set(self, p_dim, s_dim=None, dimensions=None, extant=set()):
        """
        Return a dict that describes the combination of one or two dimensions, for a plot

        :param p_dim:
        :param s_dim:
        :param dimensions:
        :param extant:
        :return:
        """

        if not dimensions:
            dimensions = self.primary_dimensions

        key = p_dim.name

        if s_dim:
            key += '/' + s_dim.name

        # Ignore if the key already exists or the primary and secondary dims are the same
        if key in extant or p_dim == s_dim:
            return

        # Don't allow geography to be a secondary dimension. It must either be a primary dimension
        # ( to make a map ) or a filter, or a small-multiple
        if s_dim and s_dim.valuetype_class.is_geo():
            return

        extant.add(key)

        filtered = {}

        for d in dimensions:
            if d != p_dim and d != s_dim:
                filtered[d.name] = d.pstats.uvalues.keys()

        if p_dim.valuetype_class.is_time():
            value_type = 'time'
            chart_type = 'line'
        elif p_dim.valuetype_class.is_geo():
            value_type = 'geo'
            chart_type = 'map'
        else:
            value_type = 'general'
            chart_type = 'bar'

        return dict(
            key=key,
            p_dim=p_dim.name,
            p_dim_type=value_type,
            p_label=p_dim.label_or_self.name,
            s_dim=s_dim.name if s_dim else None,
            s_label=s_dim.label_or_self.name if s_dim else None,
            filters=filtered,
            chart_type=chart_type
        )

    def enumerate_dimension_sets(self):

        dimension_sets = {}

        dimensions = self.primary_dimensions

        extant = set()

        for d1 in dimensions:

            ds = self.dimension_set(d1, None, dimensions, extant)

            if ds:
                dimension_sets[ds['key']] = ds

        for d1 in dimensions:
            for d2 in dimensions:

                if d2.cardinality >= d1.cardinality:
                    d1, d2 = d2, d1

                ds = self.dimension_set(d1, d2, dimensions, extant)

                if ds:
                    dimension_sets[ds['key']] = ds

        return dimension_sets

class ColumnProxy(PartitionProxy):
    def __init__(self, obj, partition):
        object.__setattr__(self, "_obj", obj)
        object.__setattr__(self, "_partition", partition)

MAX_LABELS = 75  # Maximum number of uniques records before it's assume that the values aren't valid labels

class PartitionColumn(ColumnProxy):
    """A proxy on the Column that links a Column to a Partition, for direct access to the stats
    and column labels"""

    def __init__(self, obj, partition):
        super(PartitionColumn, self).__init__(obj, partition)
        object.__setattr__(self, "pstats", partition.stats_dict[obj.name])

    @property
    def children(self):
        """"Return the table's other column that have this column as a parent, excluding labels"""
        for child in self.children:
            yield PartitionColumn(child, self._partition)

    @property
    def label(self):
        """"Return first child that of the column that is marked as a label"""
        for c in self.table.columns:
            if c.parent == self.name and 'label' in c.valuetype:
                return PartitionColumn(c, self._partition)

    @property
    def value_labels(self):
        """Return a map of column code values mapped to labels, for columns that have a label column

        If the column is not assocaited with a label column, it returns an identity map.

        WARNING! This reads the whole partition, so it is really slow

        """

        from operator import itemgetter

        card = self.pstats.nuniques

        if self.label:
            ig = itemgetter(self.name, self.label.name)
        elif self.pstats.nuniques < MAX_LABELS:
            ig = itemgetter(self.name, self.name)
        else:
            return {}

        label_set = set()
        for row in self._partition:
            label_set.add(ig(row))

            if len(label_set) >= card:
                break

        d = dict(label_set)

        assert len(d) == len(label_set)  # Else the label set has multiple values per key

        return d

    @property
    def cardinality(self):
        """Returns the bymber of unique elements"""

        return self.pstats.nuniques

    def __repr__(self):
        return "<{} {}>".format(self.__class__.__name__, self.name)

{'expand_to_years': 'ambry.util.datestimes.expand_to_years', 'NotFoundError': 'ambry.orm.exc.NotFoundError', 'MPRowsFile': 'ambry_sources.MPRowsFile', 'ResourceNotFoundError': 'fs.errors.ResourceNotFoundError', 'AccessError': 'ambry.exc.AccessError', 'S3ResponseError': 'boto.exception.S3ResponseError', 'FileLock': 'filelock.FileLock', 'ensure_dir_exists': 'ambry.util.ensure_dir_exists', 'call_interval': 'ambry.bundle.process.call_interval', 'DatabaseError': 'exc.DatabaseError'}.before_update

{'expand_to_years': 'ambry.util.datestimes.expand_to_years', 'NotFoundError': 'ambry.orm.exc.NotFoundError', 'MPRowsFile': 'ambry_sources.MPRowsFile', 'ResourceNotFoundError': 'fs.errors.ResourceNotFoundError', 'AccessError': 'ambry.exc.AccessError', 'S3ResponseError': 'boto.exception.S3ResponseError', 'FileLock': 'filelock.FileLock', 'ensure_dir_exists': 'ambry.util.ensure_dir_exists', 'call_interval': 'ambry.bundle.process.call_interval', 'DatabaseError': 'exc.DatabaseError'}.before_insert

{'expand_to_years': 'ambry.util.datestimes.expand_to_years', 'NotFoundError': 'ambry.orm.exc.NotFoundError', 'MPRowsFile': 'ambry_sources.MPRowsFile', 'ResourceNotFoundError': 'fs.errors.ResourceNotFoundError', 'AccessError': 'ambry.exc.AccessError', 'S3ResponseError': 'boto.exception.S3ResponseError', 'FileLock': 'filelock.FileLock', 'ensure_dir_exists': 'ambry.util.ensure_dir_exists', 'call_interval': 'ambry.bundle.process.call_interval', 'DatabaseError': 'exc.DatabaseError'}.before_delete


import argparse

class HackAssembler:
    def __init__(self):
        self.__comp_code = {
            "0": "0101010",
            "1": "0111111",
            "-1": "0111010",
            "D": "0001100",
            "A": "0110000",
            "!D": "0001101",
            "!A": "0110001",
            "-D": "0001111",
            "-A": "0110011",
            "D+1": "0011111",
            "A+1": "0110111",
            "D-1": "0001110",
            "A-1": "0110010",
            "D+A": "0000010",
            "D-A": "0010011",
            "A-D": "0000111",
            "D&A": "0000000",
            "D|A": "0010101",
            "M": "1110000",
            "!M": "1110001",
            "-M": "1110011",
            "M+1": "1110111",
            "M-1": "1110010",
            "D+M": "1000010",
            "D-M": "1010011",
            "M-D": "1000111",
            "D&M": "1000000",
            "D|M": "1010101",
        }
        self.__jump_code = ["", "JGT", "JEQ", "JGE", "JLT", "JNE", "JLE", "JMP"]
        self.__defined_symbols = {
            "SP": 0,
            "LCL": 1,
            "ARG": 2,
            "THIS": 3,
            "THAT": 4,
            "R0": 0,
            "R1": 1,
            "R2": 2,
            "R3": 3,
            "R4": 4,
            "R5": 5,
            "R6": 6,
            "R7": 7,
            "R8": 8,
            "R9": 9,
            "R10": 10,
            "R11": 11,
            "R12": 12,
            "R13": 13,
            "R14": 14,
            "R15": 15,
            "SCREEN": 0x4000,
            "KBD": 0x6000,
        }

    def translate(self, lines: list[str]) -> list[str]:
        return self.__handle_instructions(
            self.__handle_symbols(self.__handle_spaces(self.__handle_comments(lines)))
        )

    def __handle_symbols(self, lines: list[str]) -> list[str]:
        symbols = self.__defined_symbols.copy()
        results: list[str] = []
        for line in lines:
            if line[0] == "(" and line[-1] == ")":
                symbols[line[1:-1]] = len(results)
            else:
                results.append(line)
        counter = 16
        for (idx, line) in enumerate(results):
            if self.__is_a_instruction(line):
                value: str = line[1:]
                if value.isdigit():
                    continue
                if value not in symbols:
                    symbols[value] = counter
                    counter += 1
                results[idx] = line[0] + str(symbols[value])
        return results

    def __translate_dest(self, line: str) -> str:
        result = 0
        if "=" in line:
            dest = line.split("=")[0]
            if "M" in dest:
                result |= 1
            if "D" in dest:
                result |= 1 << 1
            if "A" in dest:
                result |= 1 << 2
        return format(result, "03b")

    def __translate_comp(self, line: str) -> str:
        st = line.index("=") + 1 if "=" in line else 0
        nd = line.index(";") if ";" in line else len(line)
        comp = line[st:nd]
        return self.__comp_code.get(comp)

    def __translate_jump(self, line: str) -> str:
        jump = None
        if ";" in line:
            jump = line.split(";")[1]
        result = self.__jump_code.index(jump or "")
        return format(result, "03b")

    def __is_a_instruction(self, line: str) -> bool:
        return line[0] == "@"

    def __translate_a_instruction(self, line: str) -> str:
        return "0" + format(int(line[1:]), "015b")[-15:]

    def __translate_c_instruction(self, line: str) -> str:
        return (
            "111"
            + self.__translate_comp(line)
            + self.__translate_dest(line)
            + self.__translate_jump(line)
        )

    def __handle_instructions(self, lines: list[str]) -> list[str]:
        result: list[str] = []
        for line in lines:
            if self.__is_a_instruction(line):
                result.append(self.__translate_a_instruction(line))
            else:
                result.append(self.__translate_c_instruction(line))
        return result

    def __handle_spaces(self, lines: list[str]) -> list[str]:
        return ["".join(line.split()) for line in lines if line.strip()]

    def __handle_comments(self, lines: list[str]) -> list[str]:
        return [line.split("//")[0] for line in lines]

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="Translate Hack assembly code to Hack binary code"
    )
    parser.add_argument("asm", help="filepath of Hack assembly code")
    args = parser.parse_args()

    filepath: str = args.asm
    assert filepath.endswith(".asm"), f"{filepath} doesn't end with .asm"

    output: str = filepath.rstrip(".asm") + ".hack"
    assembler: HackAssembler = HackAssembler()
    with open(filepath, "r") as input_file:
        code: list[str] = input_file.read().splitlines()
        with open(output, "w") as output_file:
            output_file.write("\n".join(assembler.translate(code)))


import binascii
import json
import logging

import cv2
import numpy as np
import requests

VIDEO_UPLOAD_URL = 'http://video-fs.like.video/upload_video.php'
IMAGE_UPLOAD_URL = 'http://img-fs.like.video/FileuploadDownload/upload_img.php'

logger = logging.getLogger(__name__)

def upload_video(video_bytes):
    files = {
        'file': video_bytes
    }

    try:
        resp = requests.post(VIDEO_UPLOAD_URL, files=files)
        if resp.status_code == 200:
            url = json.loads(resp.text)['url']
            crc = binascii.crc32(video_bytes)
            url = '{}?crc={}&type=5'.format(url, crc)
            return url
        else:
            return None
    except Exception as err:
        logger.error('upload_video failed, error info {}'.format(err))
        return None

def upload_image(image_bytes, req_name='default', ext='.jpg'):
    files = {
        'file': ('image{}'.format(ext), image_bytes)
    }

    try:
        if req_name == 'bigo_live':
            resp = requests.post('http://snapshot.calldev.bigo.sg/upload_file.php', files=files)
        else:
            resp = requests.post(IMAGE_UPLOAD_URL, files=files)

        if resp.status_code == 200:
            return json.loads(resp.text)['url']
        else:
            return None
    except Exception as err:
        logger.error('upload_image failed, error info {}'.format(err))
        return None

def download_video(video_url):
    try:
        resp = requests.get(video_url)
    except Exception as err:
        logger.error('download_video failed, video_url {}, error info {}'.format(video_url, err))
        return None

    if resp.status_code != 200 or not resp.content:
        logger.error('download_video failed, video_url {}'.format(video_url))
        return None

    video_bytes = resp.content

    if video_bytes is None:
        logger.error('download_video failed, empty video, video_url {}'.format(video_url))
        return None

    return video_bytes

def download_image(image_url, decode=False, to_rgb=False):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:70.0) Gecko/20100101 Firefox/70.0'
    }

    try:
        resp = requests.get(image_url, headers=headers)
    except Exception as err:
        logger.error('download_image failed, image_url {}, error info {}'.format(image_url, err))
        return None

    if resp.status_code != 200 or not resp.content:
        logger.error('download_image failed, image_url {}'.format(image_url))
        return None

    image_bytes = resp.content
    image = cv2.imdecode(np.frombuffer(image_bytes, np.uint8), cv2.IMREAD_COLOR)

    if image is None:
        logger.error('download_image failed, empty image, image_url {}'.format(image_url))
        return None

    if decode and to_rgb:
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)

    return image if decode else image_bytes

if __name__ == '__main__':
    image_bytes = requests.get('http://img.like.video/asia_live/4h6/1Jgvll.jpg').content
    image_url = upload_image(image_bytes)
    print(image_url)

    video_bytes = requests.get(
        'http://video.like.video/asia_live/7h4/M0B/C9/D7/bvsbAF37MUGEev_7AAAAAGsyOC8464.mp4').content
    video_url = upload_video(video_bytes)
    print(video_url)

    img = download_image('http://img.like.video/asia_live/4h6/1Jgvll.jpg', decode=True)
    print(img.shape)


import os
import re
import xlwt
from tkinter import *
from tkinter.filedialog import askdirectory
from xlwt import Workbook
root=Tk()
root.withdraw()
path=askdirectory()
print(path)
file_names=os.listdir(str(path))
name_lists=[]
output=[]
for file in file_names:
    file_path=path + '/' + file
    f=open(file_path,'r',encoding='utf-8',errors='ignore')
    data=f.read()
    data=data.strip()
    mhs=re.search(r'Summary((.|\s)*)',data)
    if mhs:
        data=mhs.group(1)
        rows=data.split('\n')
        for row in rows:
            if row.strip() and row.strip()!='|':
                mhs=re.search(r'(([a-zA-Z]+\s)+)\s+.*?(\d+\.\d+)',row)
                if mhs:
                    output.append([file,mhs.group(1),mhs.group(3)])
    else:
        output.append([file,'N/A','N/A'])
book=Workbook(encoding='utf-8')
sht1=book.add_sheet('sheet1')
for i in range(len(output)):
    for j in range(len(output[1])):
        sht1.write(i,j,output[i][j])
book.save('./output_data.xls')
print("over")


import Dataset as datos
import matplotlib.pyplot as plt
import numpy as np
import os

df_ventas = datos.get_df_ventas()
resample_meses = datos.get_resample_meses()

facturacion_por_juego = datos.get_facturacion_por_juego()
cantidad_ventas_por_juego = datos.get_cantidad_ventas_por_juego()
#----------------------------------------------------------------------------------------------------------------- guardo los datos en Excel
datos.guardar_en_excel(datos.get_facturacion_por_juego())
datos.guardar_en_excel(datos.get_cantidad_ventas_por_juego())
#----------------------------------------------------------------------------------------------------------------- imprimo datos en consola de python
print(facturacion_por_juego)
print(cantidad_ventas_por_juego)
#----------------------------------------------------------------------------------------------------------------- agrego descripcion de los juegos
a = datos.get_facturacion_por_juego().reset_index()
a.set_index("descripcion", inplace=True)
a.name = "Comparación de la facturación de cada juego "
#----------------------------------------------------------------------------------------------------grafico comparación facturacion de los juegos
plt.figure(figsize=[11,6]).suptitle("Comparación facturación (neta) de cada juego:")
plt.subplots_adjust(bottom=0.34, right=0.99, left=0.1, top=0.95)
plt.ylabel(a.columns[1] + " en $")
f1 = plt.bar(a.index, a["facturacion neta"], tick_label=a.index)
plt.grid(which="major", axis="y", color="black", alpha=0.15)
plt.axhline(y=a["facturacion neta"].mean(),ls="--", label= "Promedio: $" +
        "{:,}".format(round(a["facturacion neta"].mean(),2)).replace(',','x').replace('.',',').replace('x','.')+
        " (no muy útil porque se comparan \n todos los juegos, que son muy distintos)")
plt.xticks( rotation=90)
plt.yticks(np.arange(0,a["facturacion neta"].max()*1.1,datos.escala_grafico(a["facturacion neta"].max())))
plt.ticklabel_format(axis="y",style="plain", useLocale=True,)
plt.legend(loc="upper right")
axes = plt.gca()
axes.set_ylim([0,a["facturacion neta"].max()*1.1])

plt.savefig("Gráficos generales/"+a.name+".jpg")
plt.show()
plt.close()
#--------------------------------------------------------------------------------------- grafico de juegos vendidos por mes, de todos los juegos
contador=1
for juego in df_ventas.articulo.unique():
    juego = datos.get_tabla_juegos(juego)
    plt.figure().suptitle(juego)
    plt.xlabel(resample_meses.index.name)
    plt.ylabel("Número de juegos vendidos")
    f1 = plt.bar(resample_meses.index, resample_meses[juego], width=30, tick_label=resample_meses.index.strftime('%m/%y'))
    plt.grid(which="major", axis="y", color="black", alpha=0.15)
    plt.axhline(y=resample_meses[juego].mean(), ls="--",label="Promedio: $" +
     "{:,}".format(round(resample_meses[juego].mean(), 2)).replace(',', 'x').replace('.', ',').replace('x', '.'))
    plt.xticks(rotation=45)
    plt.yticks(np.arange(0, resample_meses[juego].max() * 1.1, datos.escala_grafico(resample_meses[juego].max())))
    plt.ticklabel_format(axis="y", style="plain", useLocale=True, )
    plt.legend(loc="upper right")
    axes = plt.gca()
    axes.set_ylim([0, resample_meses[juego].max() * 1.1])

    for i in f1:
        x = i.get_x()
        y = i.get_height()
        ancho = i.get_width()
        plt.text(x + ancho / 2, 0, y, fontsize=10, color="black", ha="center")
    print(contador," Se guardó " + juego+".jpg" )
    contador+=1
    plt.savefig("Gráfico de cada juego/"+juego+".jpg")
    plt.close()
del contador

#-------------------------------------------------------------------------------------------------------------------------------- GUI image viewer
from tkinter import *
from PIL import ImageTk, Image

root = Tk()
root.title('CIENCIAS PARA TODOS - Estadísticas')

image_list = []
for foto in os.listdir("Gráfico de cada juego/"):
    aux = ImageTk.PhotoImage(Image.open("Gráfico de cada juego/"+foto))
    image_list.append(aux)

my_label = Label(image=image_list[0])
my_label.grid(row=0, column=0, columnspan=3)

def forward(image_number):
    global my_label
    global button_forward
    global button_back

    my_label.grid_forget()
    my_label = Label(image=image_list[image_number - 1])
    button_forward = Button(root, text=">>", command=lambda: forward(image_number + 1))
    button_back = Button(root, text="<<", command=lambda: back(image_number - 1))

    if image_number == len(image_list):
        button_forward = Button(root, text=">>", state=DISABLED)

    my_label.grid(row=0, column=0, columnspan=3)
    button_back.grid(row=1, column=0)
    button_forward.grid(row=1, column=2)

def back(image_number):
    global my_label
    global button_forward
    global button_back

    my_label.grid_forget()
    my_label = Label(image=image_list[image_number - 1])
    button_forward = Button(root, text=">>", command=lambda: forward(image_number + 1))
    button_back = Button(root, text="<<", command=lambda: back(image_number - 1))

    if image_number == 1:
        button_back = Button(root, text="<<", state=DISABLED)

    my_label.grid(row=0, column=0, columnspan=3)
    button_back.grid(row=1, column=0)
    button_forward.grid(row=1, column=2)

button_back = Button(root, text="<<", command=back, state=DISABLED)
button_exit = Button(root, text="Exit Program", command=root.quit)
button_forward = Button(root, text=">>", command=lambda: forward(2))

button_back.grid(row=1, column=0)
button_exit.grid(row=1, column=1)
button_forward.grid(row=1, column=2)

root.mainloop()


import json
import os
import string
import requests
from selenium import webdriver
from selenium.webdriver.common.by import By
import urllib.request

options = webdriver.ChromeOptions()
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
options.add_experimental_option("prefs", {
                                "profile.password_manager_enabled": False, "credentials_enable_service": False, 'profile.default_content_settings.popups': 0})

def initCheckPoint() -> None:
    with open('checkPoint.json', 'r') as f:
        checkPoint = json.load(f)
    for key in checkPoint:
        checkPoint[key] = False
    with open('checkPoint.json', 'w') as f:
        json.dump(checkPoint, f)
    return None

def loadCheckPoint() -> dict:
    with open('checkPoint.json', 'r') as f:
        checkPoint = json.load(f)
    return checkPoint

def saveCheckPoint(checkPoint: dict) -> None:
    with open('checkPoint.json', 'w') as f:
        json.dump(checkPoint, f)
    return None

def main():
    # initCheckPoint()

    url = 'https://judgegirl.csie.org/problems/domain/0'

    # load dict of checkPoint in json file
    checkPoint = loadCheckPoint()

    # put r in a text file
    if not checkPoint['get problemSet']:
        with open('text.txt', 'w', encoding='UTF-8') as f:
            r = requests.get(url)
            # encode r into string
            f.write(r.text)

        # read text file
        # save all text between <li class="pure-menu-item" and
 into a another text file
        with open('text.txt', 'r', encoding='UTF-8') as f:
            with open('problemSet.txt', 'w', encoding='UTF-8') as f2:
                for line in f:
                    if '<li class="pure-menu-item" data=' in line:
                        while '
' not in line:
                            f2.write(line)
                            line = f.readline()
                        f2.write(line)

        # delete text file
        os.remove('text.txt')

        # read problemSet.txt and save all problemSet id into a another text file
        with open('problemSet.txt', 'r', encoding='UTF-8') as f:
            with open('problemSetId.txt', 'w', encoding='UTF-8') as f2:
                for line in f:
                    if 'data="' in line:
                        f2.write(line[line.find('#'):-3])
                        f2.write('  ')
                        continue
                    if '
' in line:
                        # fine the first alpha index
                        start = 0
                        for i in range(len(line)):
                            if line[i].isalpha():
                                start = i
                                break
                        # replace all '/' with ''
                        target = line[start:-10]
                        target = target.replace('/', '')

                        if target.find('  ') > 4:
                            continue

                        f2.write(target)
                        f2.write('\n')

        # delete problemSet.txt
        os.remove('problemSet.txt')
        checkPoint['get problemSet'] = True
        saveCheckPoint(checkPoint)

    # read problemSetId.txt
    if not checkPoint['get problemId']:
        with open('problemSetId.txt', 'r', encoding='UTF-8') as f:
            # create a folder named the text after '  ' of each line
            for line in f:
                # test if the folder already exists

                folderName = line[line.find('  ')+2:-1]

                if not os.path.exists(folderName):
                    os.mkdir(folderName)

                driver = webdriver.Chrome(
                    'chromedriver.exe', chrome_options=options)
                driver.minimize_window()
                # create a file named Description.txt in each folder
                with open(folderName+'/Description.txt', 'w', encoding='UTF-8') as f2:
                    # goto the problemSet page
                    url = 'https://judgegirl.csie.org/problems/domain/0' + \
                        line[0:line.find('  ')]
                    driver.get(url)
                    # wait for the page to load
                    # driver.implicitly_wait(3)
                    elements = driver.find_elements(
                        By.CLASS_NAME, 'problem-item')
                    # write element into Description.txt
                    for element in elements:
                        print(element.text)
                        f2.write(element.text)
                        f2.write('\n')

                driver.close()
                driver.quit()
        checkPoint['get problemId'] = True
        saveCheckPoint(checkPoint)

    url = 'https://judgegirl.csie.org/problem/0'

    if not checkPoint['get problems']:
        # go to each problemSet folder in problemSetId.txt
        folderNames = list()
        with open('problemSetId.txt', 'r', encoding='UTF-8') as f:
            for line in f:
                folderNames.append(line[line.find('  ')+2:-1])

        problemSets = dict()
        for folderName in folderNames:
            with open(folderName + '/Description.txt', 'r', encoding='UTF-8') as f:
                problemSets[folderName] = list()
                for line in f:
                    problemSets[folderName].append(line[0:-1])

        for key in problemSets:
            for problem in problemSets[key]:
                title = problem
                id = problem[0:problem.find(' ') - 1]
                # if the last character of title is a punctuation, delete it
                if title[-1] in string.punctuation:
                    title = title[:-1]
                # replace all '/' with ' '
                title = title.replace('/', ' ')
                if title in checkPoint and checkPoint[title]:
                    continue

                print('Now downloading ' + title)

                driver = webdriver.Chrome(
                    'chromedriver.exe', chrome_options=options)
                driver.minimize_window()
                # goto the problem page
                url = 'https://judgegirl.csie.org/problem/0/' + id
                driver.get(url)
                # wait for the page to load
                # driver.implicitly_wait(3)

                if not os.path.exists(key+'/'+title):
                    os.mkdir(key+'/'+title)
                with open(key+'/'+title+'/Description.txt', 'w', encoding='UTF-8') as f:
                    f.write(driver.find_element(By.CLASS_NAME, 'content').text)

                # open the file with read and write permission
                with open(key+'/'+title+'/Description.txt', 'r', encoding='UTF-8') as f:
                    start = end = 0
                    for index, line in enumerate(f):
                        if 'Task Description' in line:
                            start = index
                        if 'Submit' in line:
                            end = index
                            break

                    # goto th top of the file
                    f.seek(0)
                    # save the line between start and end into newFile
                    newFile = f.readlines()[start:end]

                with open(key+'/'+title+'/Description.txt', 'w', encoding='UTF-8') as f2:
                    # clear the file
                    f2.truncate()
                    # write the new file
                    for line in newFile:
                        f2.write(line)

                # find the photo and save it
                photos = driver.find_elements(
                    By.CLASS_NAME, 'pure-img-responsive')
                if len(photos) != 0:
                    for index, photo in enumerate(photos):
                        link = photo.get_attribute('src')
                        # save the phot
                        if 'https://judgegirl.csie.org/images/problems/' in link:
                            urllib.request.urlretrieve(link, key+'/'+title+'/'+link[44:])

                if not os.path.exists(key+'/'+title+'/testCases'):
                    os.mkdir(key+'/'+title+'/testCases')
                # find the elements that inside text is 'Download Testdata' by XPATH
                link = 'https://judgegirl.csie.org/testdata/download/' + id
                print(title)
                print(url)
                print(link)
                # goto the testCase page
                driver.get(link)
                # wait for the page to load
                # driver.implicitly_wait(3)

                content = driver.find_element(By.CLASS_NAME, 'content')
                menu = content.find_element(By.CLASS_NAME, 'pure-g')
                cases = menu.find_elements(By.CLASS_NAME, 'pure-menu-link')

                # download each testCase
                for case in cases:
                    url = case.get_attribute('href')
                    file = requests.get(url)
                    with open(key+'/'+title+'/testCases/'+case.text, 'wb') as f:
                        f.write(file.content)

                driver.close()
                driver.quit()

                checkPoint[title] = True
                saveCheckPoint(checkPoint)
        checkPoint['get problems'] = True
        saveCheckPoint(checkPoint)

    # check if all checkPoint is True
    for key in checkPoint:
        if not checkPoint[key]:
            print(key+' is not finished')
            return None
    print('all checkPoint is finished')
    return None

if __name__ == '__main__':
    main()


# encoding=utf8
import requests
from lxml import etree

class cityAreaCode():
    def __init__(self):
        self.url = "http://www.ip33.com/area/2019.html"

    def get(self):
        page = requests.get(self.url)
        page.encoding = 'utf-8'
        _element = etree.HTML(page.text)
        divs = _element.xpath('//div[@class="ip"]')[1:]
        for div in divs:
            divtext = etree.tostring(div, encoding="utf-8", pretty_print=False).decode("utf-8")
            _element1 = etree.HTML(divtext)
            h4s = _element1.xpath('//div/h4')
            lis = _element1.xpath('//div/ul/li')
            for li in lis:
                litext = etree.tostring(li, encoding="utf-8", pretty_print=False).decode("utf-8")
                _element2 = etree.HTML(litext)
                h5s = _element2.xpath('//li/h5')
                lilis = _element2.xpath('//li/ul/li')
                for lili in lilis:
                    print('（省名称）',h4s[0].text.replace(' ','（省代码）'), '（市/区名称）', h5s[0].text.replace(' ','（市/区代码）'), "（地区名称）", lili.text.replace(' ','（地区代码）'))

if __name__ == '__main__':
    cityAreaCode = cityAreaCode()
    cityAreaCode.get()


from functools import wraps
from flask import url_for, redirect, session

# 登陆限制装饰器
# 用于需要登陆的页面，如果没有登陆则要求登陆(跳转至登陆页面)
def login_required(func):
    @wraps(func)
    def wapper(*args, **kwargs):
        if session.get('user_id'):
            return func(*args, **kwargs)
        else:
            return redirect(url_for('login'))

    return wapper


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
import glob
import logging
import argparse

from ngsmetavirus.config import *
from ngsmetavirus.common import check_path, mkdir, read_tsv
from dagflow import DAG, Task, do_dag

LOG = logging.getLogger(__name__)
__version__ = "1.0.0"
__author__ = ("Xingguo Zhang",)
__email__ = "invicoun@foxmail.com"
__all__ = []

def create_merge_data_task(prefix, read1, read2, work_dir, job_type="local"):

    if len(read1)==1:
        comm1 = """\
ln -s {read1} {prefix}.raw.R1.fq.gz
ln -s {read2} {prefix}.raw.R2.fq.gz
""".format(read1=read1[0], read2=read2[0], prefix=prefix)
    else:
        comm1 = """\
cat {read1} >{prefix}.raw.R1.fq.gz
cat {read2} >{prefix}.raw.R2.fq.gz
""".format(read1=" ".join(read1), read2=" ".join(read2), prefix=prefix)

    task = Task(
        id="merge_data_%s" % prefix,
        work_dir=work_dir,
        type=job_type,
        option="-pe smp 1",
        script="""
{comm1}
""".format(comm1=comm1)
    )
    read1 = os.path.join(work_dir, "%s.raw.R1.fq.gz" % prefix)
    read2 = os.path.join(work_dir, "%s.raw.R2.fq.gz" % prefix)

    return task, read1, read2

def create_mngs_task(prefix, read1, read2, reference, nohost, dtype, atype,
                     job_type, work_dir, out_dir, trim=0, project="", id=""):

    if nohost:
        nohost = "--nohost"
    else:
        nohost = ""
    if reference:
        rx = "--reference %s" % reference
    else:
        rx = ""

    task = Task(
        id="mngs_%s" % prefix,
        work_dir=work_dir,
        type="local",
        option="-pe smp 1",
        script="""
{root}/ngsmetavirus.py all \\
  --prefix {prefix} --read1 {read1} --read2 {read2} \\
  --dtype {dtype} --atype {atype} {rx} \\
  --project {project} --id {id} \\
  --trim {trim} --thread 6 --job_type {job_type} {nohost} \\
  --work_dir {work}  --out_dir {out}
""".format(root=ROOT,
            prefix=prefix,
            read1=read1,
            read2=read2,
            rx=rx,
            nohost=nohost,
            dtype=dtype,
            atype=atype,
            trim=trim,
            project=project,
            id=id,
            job_type=job_type,
            work=work_dir,
            out=out_dir
        )
    )

    return task

def run_mngs_multi(input, reference, nohost, dtype, atype, trim, job_type,
                   concurrent, refresh, work_dir, out_dir, project="", id=""):

    input = check_path(input)
    work_dir = mkdir(work_dir)
    out_dir = mkdir(out_dir)

    if not reference:
        pass
    else:
        try:
            reference = check_path(check_path)
        except:
            reference = check_path("%s.3.ht2" % reference)
            reference = reference.strip(".3.ht2")
        else:
            raise Exception("Reference genome %s does not exist" % reference)

    data = {}

    for line in read_tsv(input):
        if line[0] not in data:
            data[line[0]] = [[], []]
        data[line[0]][0].append(check_path(line[1]))
        data[line[0]][1].append(check_path(line[2]))

    dag = DAG("mngs_multi")

    for prefix in data:

        prefix_work = mkdir(os.path.join(work_dir, prefix))
        data_task, read1, read2 = create_merge_data_task(
            prefix=prefix,
            read1=data[prefix][0],
            read2=data[prefix][1],
            work_dir=prefix_work
        )
        dag.add_task(data_task)
        task = create_mngs_task(
            prefix=prefix,
            read1=read1,
            read2=read2,
            reference=reference,
            nohost=nohost,
            dtype=dtype,
            atype=atype,
            trim=trim,
            job_type=job_type,
            project=project,
            id=id,
            work_dir=prefix_work,
            out_dir=mkdir(os.path.join(out_dir, prefix))
        )
        dag.add_task(task)
        task.set_upstream(data_task)

    do_dag(dag, concurrent, refresh)

    return 0

def mngs_multi(args):

    run_mngs_multi(
        input=args.input,
        reference=args.reference,
        nohost=args.nohost,
        dtype=args.dtype,
        atype=args.atype,
        trim=args.trim,
        work_dir=args.work_dir,
        out_dir=args.out_dir,
        concurrent=args.concurrent,
        refresh=args.refresh,
        job_type=args.job_type,
        project=args.project,
        id=args.id
    )

def add_mngs_multi_args(parser):

    parser.add_argument("input", metavar='FILE', type=str,
        help="Input the reads list.")
    parser.add_argument("-ref", "--reference", metavar="FILE", type=str, default="",
        help="Input the host's reference database.")
    parser.add_argument('--nohost', action='store_true',
        help='Input the reference database is not the host.')
    parser.add_argument("-dt", "--dtype", metavar='STR', type=str,
        choices=["mgi", "illumina", "other"], default="illumina",
        help="Set up the sequencing platform of the data, default=illumina.")
    parser.add_argument("-at", "--atype", metavar='STR', type=str,
        choices=["metagenome", "metaviral", "rnaviral"], default="metagenome",
        help="""Set the type of analysis(metagenome, metavirus, rnaviral),\
              default=metagenome.""")
    parser.add_argument("--trim", metavar="INT", type=int, default=5,
        help="Set trim length, default=5")
    parser.add_argument("--project", metavar="STR", type=str, required=True,
        help="Input project name.")
    parser.add_argument("--id", metavar="STR", type=str, required=True,
        help="Input project id.")
    parser.add_argument("--concurrent", metavar="INT", type=int, default=10,
        help="Maximum number of jobs concurrent  (default: 10).")
    parser.add_argument("--refresh", metavar="INT", type=int, default=30,
        help="Refresh time of log in seconds  (default: 30).")
    parser.add_argument("--job_type", choices=["sge", "local"], default="local",
        help="Jobs run on [sge, local]  (default: local).")
    parser.add_argument("--work_dir", metavar="DIR", type=str, default=".",
        help="Work directory (default: current directory).")
    parser.add_argument("--out_dir", metavar="DIR", type=str, default=".",
        help="Output directory (default: current directory).")

    return parser

def main():

    logging.basicConfig(
        stream=sys.stderr,
        level=logging.INFO,
        format="[%(levelname)s] %(message)s"
    )

    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description="""
attention:
    ngsmetavirus.py.py multi input.list
File format：
name    R1  R2

version: %s
contact:  %s <%s>\
    """ % (__version__, " ".join(__author__), __email__))

    parser = add_mngs_multi_args(parser)
    args = parser.parse_args()
    mngs_multi(args)

if __name__ == "__main__":
    main()


import numpy as np
from scipy.constants import G
from scipy.interpolate import interp1d
from astropy.constants import kpc

import general as ge

class zhao(object):
    """ Class for generating a potential for a spherical dark matter halo,
        using the data generated by the model of Zhao (2009).

        Attributes:

            fName:  The name of the file containing the data generated by the
                    model of Zhao. The data should be generated using the code
                    provided at: http://202.127.29.4/dhzhao/mandc.html

            data:   The loaded data from the text file; the data is stored in
                    a 2D numpy array. For more information on which data is
                    stored, see the README file generated using the code.

            h0:     Little h, hubble parameter at z=0 divided by 100, unitless.
                    Value from the Planck collaboration (2020).

            red:    The redshift values at which the properties of the dark
                    matter halo are computed.

            mass:   The virial mass of the dark matter halo in units of solar
                    masses, computed at the redshifts given by red.

            conc:   The concentration of the halo (r_vir / r_s) at the given
                    redshifts.

            virR:   The virial radius at the redshift values of the dark
                    matter halo in meters.

            rhoS:   The density at the scale radius in kg/m^3.

            rS:     The scale radius of the dark matter halo in meters.

            time:   The lookback time corresponding to the redshift values in
                    years.
    """

    def __init__(self, fName):
        """ Initializing the potential according to the model of Zhao

            Input:
                fName   (string)

            Returns:
                zhao    (object)
        """

        self.fName = fName                      # File name

        data = self.load_data()
        h0 = 0.6766
        t0 = data[:,-1][0] / h0

            # Unpacking data
        self.red = data[:,0]                            # Redshift
        self.mass = data[:,1] / h0                      # Virial mass
        self.conc = data[:,2]                           # Concentration
        self.virR = 1e3 * kpc.value * data[:,4] / h0    # Virial radius
        self.rhoS = ge.conv_inv_dens(data[:,7]/1e18) * h0 * h0   # rho_s
        self.rS = 1e3 * kpc.value * data[:,8] / h0      # r_s
        self.time = t0 - data[:,-1] / h0                # Age of Universe

    def load_data(self):
        """ Loading data from a generated data file

            Input:
                -

            Returns:
                data:   array containing the properties of the dark matter
                        halo (2D numpy array).
        """
        with open(self.fName) as f:
            data = np.loadtxt((x.replace('   ', '  ') for x in f), skiprows=1)
        return data

    def find_z_ind(self, zV):
        """ Input a redshift and find the indices corresponding to the closest
            redshift value(s) of the generated data. If zV is an array then
            the closest indices for all the redshift values are determined and
            returned.

            Input:
                zV:     The redshift value(s) for which the closest index has
                        to be found (float or numpy array).

            Returns:
                The indices corresponding to the closest redshift values
                (integer or numpy array).
        """

        if type(zV) != np.ndarray and type(zV) != list and type(zV) != tuple:
            return ge.find_closest(self.red, zV)[0]

        return np.asarray([ge.find_closest(self.red, z)[0] for z in zV])

    def rs_rhos_at_z(self, zV):
        """ Find the scale radius (r_s) and the density at the scale radius
            (rho_s) for a given redshift value. This is done by finding the
            closest redshift value to the input redshift value(s), NOT by
            interpolating.

            Input:
                zV:     redshift(s) at which r_s and rho_s will be determined
                        (float or numpy array).

            Returns:
                Value of r_s at zV (float or numpy array).
                Value of rho_s at zV (float or numpy array).
        """
        zInd = self.find_z_ind(zV)                      # Correct z index
        return self.rS[zInd], self.rhoS[zInd]

    def mass_at_r(self, zV, r):
        """ The mass of the dark matter halo as function of distance from the
            center of the dark matter halo at a given redshift.

            Input:
                zV:     redshift(s) at which the mass as function of radius
                        is determined (float or numpy array).
                r:      the distances from the center of the halo at which
                        the mass will be calculated (float or numpy array).

            Returns:
                mass as function of r and z (float or numpy array (1D or 2D))
        """

        rS, rhoS = self.rs_rhos_at_z(zV)                # r_s and rho_s

        mass = [rhoS[i] * rS[i] * (np.log(1+r/rS[i]) - r / (rS[i] + r))
                for i in range(len(rS))]

        return 16 * np.pi * np.asarray(mass)

    def simp_profile(self, x):
        """ The NFW profile density profile for the dark matter halo. This
            function gives rho/rho_s as function of r/r_s. Therefore you do
            not need to specify the parameters r_s and rho_s. Moreover, this
            profile is time independent.

            Input:
                x:      r/r_s values, dimensionless (numpy array).

            Returns:
                rho/rho_s for the given x values (numpy array).
        """
        return 4 / (x * np.power(1+x, 2))

    def nfw_profile(self, zV, r):
        """ A time dependent NFW density profile. With the input of the
            desired redshift value(s), a time dependent density profile
            as function of radius is output. r_s and rho_s are determined
            using the model of van den Bosch.

            Input:
                zV:     the redshift values at which the density profile
                        is computed (float or numpy array).
                r:      distance from the center of the halo (float or
                        numpy array).

            Returns:
                time dependent NFW profile (float or numpy array(1D or 2D))
        """

        zInd = self.find_z_ind(zV)                          # Selecting z ind

        rS = self.rS                                        # r_s
        rhoS = self.rhoS                                    # rho_s

        if type(zInd) != np.ndarray:                        # Single z value
            denom = r * np.power(1 + r/rS[zInd], 2) / rS[zInd]
            return 4 * rhoS[zInd] / denom

        selrS, selrhoS = rS[zInd], rhoS[zInd]               # Slicing lists

            # Multiple z values
        frac = [selrhoS[ind] / (r * np.power(1 + r/selrS[ind], 2) / selrS[ind])
                for ind in range(len(selrS))]

        return 4 * np.asarray(frac)

    def pot_nfw(self, zV, r):
        """ The gravitational potential corresponding to the NFW
            density profile. This is obtained by solving the Poisson
            equation. For the NFW profile there exists an analytical
            solution.

            Input:
                zV:     the redshift values at which the potential is
                        computed (float or numpy array).
                r:      distance from the center of the halo (float or
                        numpy array).

            Returns:
                gravitational potential (float or numpy array(1D or 2D))
        """

        rhoS = self.rhoS                                    # rho_s
        rS = self.rS                                        # r_s

        zInd = self.find_z_ind(zV)                          # Finding correct z

        if type(zInd) != np.ndarray:                        # Single z value

               # Need to interpolate due to coarse grid
            interpRS = interp1d(self.red, self.rS)      # Creating r_s int. object
            selrS = interpRS(zV)                        # Interpolating r_s

            interpRhoS = interp1d(self.red, self.rhoS)  # Creating rho_s int. object
            selrhoS = interpRhoS(zV)                    # Interpolating rho_s

            part1 = -16 * np.pi * G * selrhoS * selrS * selrS
            part2 = np.log(1 + r/selrS) / (r/selrS)

            return part1 * part2

            # Multiple z values
        selrS, selrhoS = rS[zInd], rhoS[zInd]                   # Slicing lists

        part1 = -16 * np.pi * G * selrhoS * selrS * selrS       # First part
        part2 = [np.log(1 + r/rsV) / (r/rsV) for rsV in selrS]  # Second part

        phi = [part1[ind] * part2[ind] for ind in range(len(zInd))] # Potential

        return np.asarray(phi)


from collections import defaultdict

def solution(id_list, report, k):
    table = defaultdict(int)  # 누가 몇번 신고 받은지 체크하는 테이블
    answer = [0] * len(id_list)

    for repo in set(report):
        table[repo.split()[1]] += 1

    # k번 이상 신고 받은 사람(report.split(' ')[1])인 경우
    # 신고한 사람(report.split(' ')[0]) 메일 발송 횟수 1 추가
    for repo in set(report):
        if table[repo.split(' ')[1]] >= k:
            answer[id_list.index(repo.split(' ')[0])] += 1

    return answer


# Woman who habitually buys pastries before 5

import json

def find_customers_who_order_multiple_pastries_before_5am() -> list[str]:
    """
    Identifies customer ids of customers who
    placed orders between midnight and 5am
    """
    with open('./noahs-jsonl/noahs-orders.jsonl', 'r') as jsonl_file:

        maybe_tinder_lady_customer_id = []

        for line in jsonl_file:

            order = json.loads(line)

            date_time = order["ordered"]

            time = date_time.split(" ")[1]
            hour = time.split(":")[0]
            hour = int(hour)

            bakery_items_count = 0
            for i in range(len(order["items"])):
                if "BKY" in order["items"][i]["sku"]:
                    bakery_items_count += 1 * int(order["items"][i]["qty"])

            if hour < 5 and bakery_items_count > 1:
                maybe_tinder_lady_customer_id.append(order["customerid"])

        return maybe_tinder_lady_customer_id

def find_customers_from_customer_ids(customer_ids: list[str]) -> list[dict]:
    """
    Gets a list of customers from
    a list of customer ids
    """
    with open('./noahs-jsonl/noahs-customers.jsonl', 'r') as jsonl_file:

        maybe_tinder_lady_info = []

        for line in jsonl_file:

            customer = json.loads(line)
            customer_id = customer["customerid"]

            if customer_id in customer_ids:
                frequency = customer_ids.count(customer_id)
                customer["freq"] = frequency
                maybe_tinder_lady_info.append(customer)

        return maybe_tinder_lady_info

if __name__ == "__main__":

    possible_customer_ids = find_customers_who_order_multiple_pastries_before_5am()
    possible_customers = find_customers_from_customer_ids(possible_customer_ids)
    possible_customers = sorted(possible_customers, key=lambda x: x["freq"], reverse=True)

    for customer in possible_customers:
        # First customer to be printed is the one who most habitually
        # purchases pastries early morning
        print(customer)


import matplotlib.pyplot as plt
import numpy as np
import dill as pickle

sens_to_plot = ['20180314_grav_noshield_cant-0mV_allharm.npy', \
                '20180314_grav_shieldin-nofield_cant-0mV_allharm.npy', \
                '20180314_grav_shieldin-1V-1300Hz_cant-0mV_allharm.npy', \
                '20180314_grav_shieldin-2V-2200Hz_cant-0mV_allharm.npy']

labs = ['No Shield', 'Shield', 'Shield - 1300Hz', 'Shield - 2200Hz']

plot_just_current = False

sens_dat = []
for sens_file in sens_to_plot:
    lambdas, alphas, diagalphas = np.load('/sensitivities/' + sens_file)
    sens_dat.append(alphas)

alpha_plot_lims = (1000, 10**13)
lambda_plot_lims = (10**(-7), 10**(-4))

limitdata_path = '/sensitivities/decca1_limits.txt'
limitdata = np.loadtxt(limitdata_path, delimiter=',')
limitlab = 'No Decca 2'

limitdata_path2 = '/sensitivities/decca2_limits.txt'
limitdata2 = np.loadtxt(limitdata_path2, delimiter=',')
limitlab2 = 'With Decca 2'

fig, ax = plt.subplots(1,1,sharex='all',sharey='all',figsize=(5,5),dpi=150)

if not plot_just_current:
    for i, sens in enumerate(sens_dat):
        ax.loglog(lambdas, sens, linewidth=2, label=labs[i])

ax.loglog(limitdata[:,0], limitdata[:,1], '--', \
          label=limitlab, linewidth=3, color='r')
ax.loglog(limitdata2[:,0], limitdata2[:,1], '--', \
          label=limitlab2, linewidth=3, color='k')
ax.grid()

ax.set_xlim(lambda_plot_lims[0], lambda_plot_lims[1])
ax.set_ylim(alpha_plot_lims[0], alpha_plot_lims[1])

ax.set_xlabel('$\lambda$ [m]')
ax.set_ylabel('|$\\alpha$|')

ax.legend(numpoints=1, fontsize=9)

#ax.set_title(figtitle)

plt.tight_layout()

plt.show()


# https://www.acmicpc.net/problem/1260

# solution
# 1) 주어진 입력을 인접행렬로 저장한다
# 2) 현재 정점 그때그때 출력하는 dfs를 돌린다
# 3) bfs 하고 path를 출력한다

# TIL
# adj_mat = [([0,] * n),]*n -> 이런식으로 초기화 하면안됨. copy라 원소의 id값 다 같아지는 문제
# 파이썬 입력으로 input()은 굉장히 느리다. sys.stdin.readline() 사용
# dfs의 경우 최단거리 찾는 문제(탐색 여러번 반복)와 단순히 탐색하는 문제(한번만 탐색) 구분해서 풀자
## 최단거리 찾는 경우만 여러번 반복해야해서 visited 리셋필요
# 반면 bfs의 path는 그 자체로 최단거리(자체적으로 여러개 path로 탐색)

# bfs 제대로 이해하자
## 1) bfs는 재귀없이 반복만
## 2) 방문과 발견 시점의 분리
## 3) 큐에 중복된 정점 안들어가게 조심
## 4) 정점 방문 표시는 반드시 큐에 넣을때 해야함

from collections import deque
import sys

def dfs(v):
    global adj_mat, visited

    print(v+1, end=' ')

    for n_v, is_conn in enumerate(adj_mat[v]): # 다음으로 방문할 next_v를 찾기위해 인접행렬의 v번째 row 순회한다
        if v == n_v:continue # 같은 v에 대해선 탐색하지 않음
        if is_conn == 1 and visited[n_v] == 0: # 현재 v와 연결된 next_v에 방문하지 않았으면 방문한다
            visited[n_v] = 1 # next_v에 방문했음을 마킹
            dfs(n_v)

def bfs(start):
    global adj_mat

    visited = [0 for _ in range(n)]
    path = deque([])

    q = deque([start,]) # 첫 정점 q에 push하며 시작
    visited[start] = 1 # q에 push 순간 방문표시

    while len(q) > 0:
        # 방문(q에서 pop하는 일)
        v = q.popleft()
        path.append(v)

        # 발견(q에 push하는 일)
        for n_v,is_conn in enumerate(adj_mat[v]):# 방문할 next_v를 찾기위해 인접행렬의 v번째 row 순회한다
            if v == n_v: continue # 같은 v에 대해선 탐색하지 않음
            if is_conn == 1 and visited[n_v] == 0: # v 다음에 방문할수있고, 현재 q에 없는 next_v 있으면
                q.append(n_v) # 발견한 next_v를 큐에 추가
                visited[n_v] = 1 # q에 push 순간 방문표시

    for v in path:
        print(v+1, end=' ')

if __name__ == "__main__":
    n,m,v = tuple(map(int,sys.stdin.readline().split()))
    v = v-1 # 인덱스로 탐색위해 정점 번호를 하나 줄여준다(v가 1부터 시작하기 때문)

    # 인접행렬 초기화 및 저장
    adj_mat = [[0 for _ in range(n)] for _ in range(n)]
    for _ in range(m):
        r,c = tuple(map(int,sys.stdin.readline().split()))
        adj_mat[r-1][c-1] = 1
        adj_mat[c-1][r-1] = 1

    # dfs
    visited = [0 for _ in range(n)]
    visited[v] = 1
    dfs(v)

    print('\n',end='')

    # bfs
    bfs(v)