#!/usr/bin/env python
# encoding: utf-8
"""
plate.py

Created by José Sánchez-Gallego on 18 Apr 2014.
Licensed under a 3-clause BSD license.

Revision history:
    18 Apr 2014 J. Sánchez-Gallego
      Initial version

"""

from __future__ import division
from __future__ import print_function
from Totoro.db import getConnectionFull
from Totoro import exceptions as TotoroExceptions
from Totoro import log, config, dustMap, site
from Totoro import utils
from Totoro.scheduler import observingPlan
from Totoro.dbclasses import Set as TotoroSet
from Totoro.dbclasses import Exposure as TotoroExposure
from Totoro.dbclasses import plate_utils as plateUtils
from Totoro.scheduler.footprint import getPlatesInFootprint
import warnings
from astropy import time
import numpy as np
from copy import deepcopy
from sqlalchemy import or_, and_
from sqlalchemy.orm.exc import NoResultFound

__all__ = ['getPlugged', 'getAtAPO', 'getAll', 'getComplete', 'Plate',
           'fromPlateID']

def getPlugged(**kwargs):

    kwargs.setdefault('fullCheck', False)
    kwargs.setdefault('updateSets', False)

    __, Session, plateDB, __ = getConnectionFull()
    session = Session()

    with session.begin():
        activePluggings = session.query(
            plateDB.ActivePlugging).join(
                plateDB.Plugging,
                plateDB.Plate,
                plateDB.PlateToSurvey,
                plateDB.Survey,
                plateDB.SurveyMode).filter(
                    plateDB.Survey.label == 'MaNGA',
                    plateDB.SurveyMode.label.ilike('MaNGA%')).order_by(
                        plateDB.Plate.plate_id).all()

    plates = [actPlug.plugging.plate for actPlug in activePluggings]

    return Plates(plates, **kwargs)

def _getValidPlates(plates, rejectLowPriority=False,
                    rejectSpecial=False, **kargs):
    """Rejects low priority and special plates."""

    minimumPriority = config['plugger']['noPlugPriority']

    if rejectLowPriority:
        plates = [plate for plate in plates
                  if plate.plate_pointings[0].priority > minimumPriority]

    validPlates = []
    if rejectSpecial:
        for plate in plates:
            if plate.mangadbPlate is None:
                continue
            if (plate.mangadbPlate.all_sky_plate is True or
                    plate.mangadbPlate.commissioning_plate is True or
                    plate.mangadbPlate.neverobserve is True):
                continue
            validPlates.append(plate)
    else:
        validPlates = plates

    return validPlates

def getAtAPO(onlyIncomplete=False, onlyMarked=False, raRange=None, **kwargs):
    """Gets plates at APO with various conditions."""

    kwargs.setdefault('fullCheck', False)
    kwargs.setdefault('updateSets', False)

    __, Session, plateDB, __ = getConnectionFull()
    session = Session()

    with session.begin():
        plates = session.query(plateDB.Plate).join(
            plateDB.PlateToSurvey, plateDB.Survey, plateDB.SurveyMode).filter(
                plateDB.Survey.label == 'MaNGA',
                plateDB.SurveyMode.label == 'MaNGA dither'
        ).join(plateDB.PlateLocation).filter(
            plateDB.PlateLocation.label == 'APO').join(
                plateDB.PlatePointing, plateDB.Pointing)

        _checkNumberPlatesAtAPO(plates)

        if onlyMarked is False:
            plates = plates
        else:
            plates = plates.join(plateDB.PlateToPlateStatus,
                                 plateDB.PlateStatus).filter(
                plateDB.PlateStatus.label == 'Accepted')

        if raRange is not None:
            if raRange.size == 2:
                plates = plates.filter(
                    plateDB.Pointing.center_ra >= raRange[0],
                    plateDB.Pointing.center_ra <= raRange[1])
            elif raRange.size == 4:
                plates = plates.filter(
                    or_(and_(plateDB.Pointing.center_ra >= raRange[0][0],
                             plateDB.Pointing.center_ra <= raRange[0][1]),
                        and_(plateDB.Pointing.center_ra >= raRange[1][0],
                             plateDB.Pointing.center_ra <= raRange[1][1])))
            else:
                warnings.warn('unrecognised format for raRange',
                              TotoroExceptions.TotoroUserWarning)

        plates = plates.order_by(plateDB.Plate.plate_id).all()

    validPlates = _getValidPlates(plates, **kwargs)

    if onlyIncomplete:
        return _getIncomplete(validPlates, **kwargs)
    else:
        return Plates(validPlates, **kwargs)

def getAll(onlyIncomplete=False, **kwargs):

    kwargs.setdefault('fullCheck', False)
    kwargs.setdefault('updateSets', False)

    __, Session, plateDB, __ = getConnectionFull()
    session = Session()

    with session.begin():
        plates = session.query(plateDB.Plate).join(
            plateDB.PlateToSurvey, plateDB.Survey, plateDB.SurveyMode
        ).filter(plateDB.Survey.label == 'MaNGA',
                 plateDB.SurveyMode.label == 'MaNGA dither').order_by(
                     plateDB.Plate.plate_id)

    platesAtAPO = plates.join(plateDB.PlateLocation).filter(
        plateDB.PlateLocation.label == 'APO')
    _checkNumberPlatesAtAPO(platesAtAPO)

    plates = plates.order_by(plateDB.Plate.plate_id).all()

    plates = _getValidPlates(plates, **kwargs)

    if onlyIncomplete:
        return _getIncomplete(plates, **kwargs)
    else:
        return Plates(plates, **kwargs)

def _getIncomplete(plates, **kwargs):

    totoroPlates = Plates(plates, **kwargs)

    incompletePlates = []
    for totoroPlate in totoroPlates:
        if not totoroPlate.isComplete:
            incompletePlates.append(totoroPlate)

    return incompletePlates

def getComplete(**kwargs):
    """Retrieves only complete files by looking at the plugging status
    (faster than using getAll and then selecting the complete plates)."""

    kwargs.setdefault('fullCheck', False)
    kwargs.setdefault('updateSets', False)

    db, Session, plateDB, mangaDB = getConnectionFull()
    session = Session()

    with session.begin():
        plates = session.query(plateDB.Plate).join(
            plateDB.PlateToSurvey, plateDB.Survey, plateDB.SurveyMode,
            plateDB.Plugging, plateDB.PluggingStatus).filter(
                plateDB.Survey.label == 'MaNGA',
                plateDB.SurveyMode.label == 'MaNGA dither',
                plateDB.PluggingStatus.label.in_(['Good', 'Overridden Good'])
        ).order_by(plateDB.Plate.plate_id).all()

    return Plates(plates, **kwargs)

def _checkNumberPlatesAtAPO(plates):
    """Raises a warning if the number of plates at APO is reaching the limit
    we can store there."""

    nPlatesAPO = config['numberPlatesAllowedAtAPO']
    nPlates = plates.count()
    if nPlates >= 0.9 * nPlatesAPO:
        warnings.warn('MaNGA has {0} plates at APO when the maximum is {1}'
                      .format(nPlates, nPlatesAPO),
                      TotoroExceptions.TotoroUserWarning)

    return

class Plates(list):

    def __init__(self, inp, format='pk', **kwargs):

        __, __, plateDB, __ = getConnectionFull()

        if all([isinstance(ii, Plate) for ii in inp]):
            list.__init__(self, inp)
        elif all([isinstance(ii, plateDB.Plate) for ii in inp]):
            list.__init__(self, [Plate(ii, **kwargs) for ii in inp])
        else:
            list.__init__(self, [Plate(ii, format=format, **kwargs)
                                 for ii in inp])

    @staticmethod
    def getPlugged(**kwargs):
        """For backards compatibility."""
        return getPlugged(**kwargs)

    @staticmethod
    def getAll(**kwargs):
        """For backards compatibility."""
        return getAll(**kwargs)

    @staticmethod
    def getAtAPO(**kwargs):
        """For backards compatibility."""
        return getAtAPO(**kwargs)

    def plate_id(self):
        """Returns a list of the plate_ids of the plates in the object."""
        return [plate.plate_id for plate in self]

    def manga_tileid(self):
        """Returns a list of the manga_tileids of the plates in the object."""
        return [plate.manga_tileid for plate in self]

def fromPlateID(plateid, **kwargs):
    """Convenience function that returns a `Plate` instance from a plate_id."""
    return Plate(input=plateid, format='plate_id', **kwargs)

class Plate(object):

    _instances = {}

    def __new__(cls, input=None, format='pk', **kwargs):

        # Checks if an open transaction already exists.
        # utils.checkOpenSession()

        me = object.__new__(cls)

        me.db, Session, plateDB, __ = getConnectionFull()
        me.session = Session()

        # Initialises DB object
        if input is None:
            me._dbObject = plateDB.Plate()
            return me
        else:
            if isinstance(input, plateDB.Plate):
                me._dbObject = input
            else:
                me._dbObject = me._initFromData(input, format=format)

        # If the DB object already exists in the library of Totoro.Plate
        # instances, returns it. Otherwise, records it and returns the new
        # object.
        if me._dbObject in cls._instances:
            return cls._instances[me._dbObject]
        else:
            cls._instances[me._dbObject] = me
            return me

    def __init__(self, input=None, format='pk', mock=False,
                 updateSets=True, mjd=None, fullCheck=True,
                 manga_tileid=None, **kwargs):
        """A custom class based on plateDB.Plate."""

        self.__dbAttributes__ = self._dbObject.__mapper__.attrs

        self._complete = None
        self._drilled = None
        self._priority = None
        self.isMock = mock
        self._kwargs = kwargs
        self._inFootprint = None
        self.mjd = mjd
        self._manga_tileid = manga_tileid
        self._mlhalimit = None

        # Date (JD) at which the plate will arrive at APO. Used for field
        # selection. If None, it assumes the plate is already at APO.
        self._dateAtAPO = None

        if 'dust' in kwargs:
            self.dust = kwargs['dust']
        else:
            self._dust = None

        if not self.isMock:
            self.sets = [TotoroSet(set, **kwargs)
                         for set in self.getMangaDBSets()]

            self.checkPlate(full=fullCheck)

            if updateSets:
                self.updatePlate(**kwargs)

        else:
            self.sets = []

    def __repr__(self):
        return ('<Totoro Plate (plate_id={0}, manga_tileid={1}, '
                'completion={2:.2f})>'
                .format(self.plate_id, self.manga_tileid,
                        self.getPlateCompletion()))

    def _initFromData(self, input, format):
        """Init a new Plate instance from a DB query."""

        with self.session.begin():
            try:
                plate = self.session.query(self.db.plateDB.Plate).filter(
                    eval('self.db.plateDB.Plate.{0} == {1}'
                         .format(format, input))).one()
            except NoResultFound:
                raise TotoroExceptions.TotoroError('no plate found for input '
                                                   '{0}={1}'
                                                   .format(format, input))

        return plate

    def __getattr__(self, name):
        """Custom getattr method that first looks into the DB object."""

        if name in object.__getattribute__(self, '__dbAttributes__'):
            return getattr(self._dbObject, name)
        else:
            return object.__getattribute__(self, name)

    def __setattr__(self, name, value):
        """Custom setattr method that first looks into the DB object."""

        if hasattr(self, '_dbObject') and hasattr(self._dbObject, name):
            setattr(self._dbObject, name, value)
        else:
            super(Plate, self).__setattr__(name, value)

    @classmethod
    def fromSets(cls, sets, **kwargs):

        newPlate = cls(None, mock=True, **kwargs)
        newPlate.sets = sets

        log.debug('created mock plate from sets pk={0}'.format(
            ', '.join(map(str, sets))))

        return newPlate

    @classmethod
    def createMockPlate(cls, ra=None, dec=None, **kwargs):

        if ra is None or dec is None:
            raise TotoroExceptions.TotoroError('ra and dec must be specified')

        mockPlate = cls(mock=True, ra=ra, dec=dec, **kwargs)
        mockPlate.isMock = True

        log.debug('created mock plate with ra={0:.3f} and dec={0:.3f}'.format(
                  mockPlate.coords[0], mockPlate.coords[1]))

        return mockPlate

    def addExposure(self, exposure):
        return plateUtils.addExposure(exposure, self)

    def getMangaDBSets(self):
        """Returns a list of mangaDB.ModelClasses.Set instances with all the
        sets matching exposures in this plate. Removes duplicates."""

        plateDB = self.db.plateDB
        mangaDB = self.db.mangaDB

        # Finds the sets for this plate
        sets = self.session.query(mangaDB.Set).distinct(mangaDB.Set.pk).join(
            mangaDB.Exposure, plateDB.Exposure, plateDB.Observation,
            plateDB.PlatePointing, plateDB.Plate).filter(
                plateDB.Plate.pk == self.pk).all()

        # sets = sorted(sets, key=lambda set: set.pk)

        return sets

    def checkPlate(self, full=True):
        """Does some sanity checks for the current plate. If full=True,
        it checks that all the science exposures have a mangaDB counterpart.
        This can be disabled to save time during plate bulk load."""

        # Checks that we are not inside an open session
        # utils.checkOpenSession()

        if not hasattr(self, 'pk') or not hasattr(self, 'plate_id'):
            raise AttributeError('Plate instance has no pk or plate_id.')

        if not self.isMaNGA:
            raise TotoroExceptions.NoMangaPlate('this is not a MaNGA plate!')

        if full:
            nMaNGAExposures = len(self.getMangadbExposures())
            nScienceExposures = len(self.getScienceExposures())
            if nMaNGAExposures != nScienceExposures:
                warnings.warn('plate_id={1}: {0} plateDB.Exposures found '
                              'but only {2} mangaDB.Exposures'.format(
                                  nScienceExposures, self.plate_id,
                                  nMaNGAExposures),
                              TotoroExceptions.NoMangaExposure)

    @property
    def isMaNGA(self):

        for survey in self.surveys:
            if survey.label == 'MaNGA':
                return True

        return False

    def updatePlate(self, force=False, **kwargs):

        # Checks if some sets have incompletely reduced exposures. If so,
        # removes their set_pk so that they are completely procesed again.
        # with session.begin():
        #     for ss in self.sets:
        #         for exp in ss.totoroExposures:
        #             if exp._mangaExposure.status is None:
        #                 if not exp.isMock:
        #                     exp.mangadbExposure[0].set_pk = None
        #                 ss.totoroExposures.remove(exp)
        #                 ss.set_status_pk = None

        #         # Checks if the sets is now empty, if so, removes it
        #         # from the object and from the DB
        #         if len(ss.totoroExposures) == 0:
        #             if not ss.isMock:
        #                 setDB = session.query(mangaDB.Set).get(ss.pk)
        #                 session.delete(setDB)
        #             self.sets.remove(ss)

        # if self.isComplete:
        #     if not force:
        #         log.debug('plate_id={0} is marked complete. '
        #                   'Not updating sets.'.format(self.plate_id))
        #         return False
        #     else:
        #         log.info('plate_id={0} is marked complete but force=True.'
        #                  .format(self.plate_id))

        if (self.currentSurveyMode is None or
                self.currentSurveyMode.label != 'MaNGA dither'):
            log.debug('plate_id={0} has surveyMode which is not MaNGA dither. '
                      'Not updating sets.'.format(self.plate_id))
            return False

        result = plateUtils.updatePlate(self, **kwargs)

        return result

    def rearrangeSets(self, LST=None, mode='optimal', scope='all', **kwargs):
        result = plateUtils.rearrangeSets(self, LST=LST, scope=scope,
                                          mode=mode, **kwargs)

        return result

    @property
    def ra(self):
        return self.getCoordinates()[0]

    @property
    def dec(self):
        return self.getCoordinates()[1]

    @property
    def coords(self):
        """Alias for getCoordinates(). Provides backwards compatibility."""
        return self.getCoordinates()

    def getCoordinates(self):

        if 'ra' in self._kwargs and 'dec' in self._kwargs:
            if (self._kwargs['ra'] is not None and
                    self._kwargs['dec'] is not None):
                return np.array(
                    [self._kwargs['ra'], self._kwargs['dec']], np.float)
        else:

            if len(self.plate_pointings) > 1:
                warnings.warn('plate_id={0:d}: multiple plate pointings found.'
                              ' Using the first one.'.format(self.plate_id),
                              TotoroExceptions.MultiplePlatePointings)

            return np.array(
                [self.plate_pointings[0].pointing.center_ra,
                 self.plate_pointings[0].pointing.center_dec], np.float)

    @property
    def isComplete(self):
        """Returns True if the plate is complete."""

        # This is a hidden option mostly to be used with
        # Totoro.scheduler.io.restoreExposures. If _useOnlyCompletion
        # is set, the plate is considered complete by just looking at
        # getPlateCompletion() (i.e., skips checking the plugging status).
        if hasattr(self, '_useOnlyCompletion'):
            return self.getPlateCompletion() > 1.

        if self._complete is not None:
            return self._complete
        else:
            status = utils.isPlateComplete(self)
            # If the plate is mock, caches the status.
            if status is True and self.isMock:
                self._complete = status
            return status

    def copy(self):
        return deepcopy(self)

    def getPlateCompletion(self, includeIncompleteSets=False, useMock=True):

        # totalSN = self.getCumulatedSN2(
        #     includeIncomplete=includeIncompleteSets, useMock=useMock)

        validStatuses = ['Good', 'Excellent', 'Override Good']
        if includeIncompleteSets:
            validStatuses.append('Incomplete')

        sets = []
        for ss in self.sets:
            if not useMock and ss.isMock:
                continue
            if ss.getStatus()[0] in validStatuses:
                sets.append(ss)

        exposureSN2 = np.array([exp.getSN2Array() for ss in sets
                                for exp in ss.totoroExposures])

        if exposureSN2.shape[0] == 0:
            return 0

        blueSN2 = np.nansum(np.nanmean(exposureSN2[:, [0, 1]], axis=1))
        redSN2 = np.nansum(np.nanmean(exposureSN2[:, [2, 3]], axis=1))

        if np.isnan(blueSN2) or np.isnan(redSN2):
            return 0
        else:
            return np.min([blueSN2 / config['SN2thresholds']['plateBlue'],
                          redSN2 / config['SN2thresholds']['plateRed']])

    def getSN2Array(self, **kwargs):
        """Same as getCumulatedSN2. Added for consistency with Set and
        Exposure."""
        return self.getCumulatedSN2(**kwargs)

    def getCumulatedSN2(self, includeIncomplete=False, useMock=True,
                        **kwargs):
        """Returns the cumulated SN2 from all valid sets.
        If includeIncomplete=True, incomplete sets SN2 are also included.
        If useMock=False, mock sets are ignored.
        """

        validStatuses = ['Good', 'Excellent', 'Override Good']
        if includeIncomplete:
            validStatuses.append('Incomplete')

        validSets = []
        for set in self.sets:
            # Creates a mock set with the appropriate exposures.
            if useMock:
                mockSet = set
            else:
                if set.isMock:
                    continue
                mockSet = TotoroSet.fromExposures(
                    [exp for exp in set.totoroExposures if not exp.isMock])
            if mockSet.getQuality()[0] in validStatuses:
                validSets.append(mockSet)

        if len(validSets) == 0:
            return np.array([0.0, 0.0, 0.0, 0.0])
        else:
            return np.nansum([set.getSN2Array(useMock=useMock)
                              for set in validSets], axis=0)

    def getActiveCartNumber(self):
        """Returns the cart number of the active plugging. Raises an error if
        no active pluggings are found."""

        for plugging in self.pluggings:
            if len(plugging.activePlugging) > 0:
                return int(plugging.cartridge.number)

        raise TotoroExceptions.PlateNotPlugged(
            'plate_id={0} is not currently plugged'.format(self.plate_id))

    def getActivePlugging(self):
        """Returns the active plugging or None if none found."""

        if self.isMock:
            return None

        for plugging in self.pluggings:
            if len(plugging.activePlugging) > 0:
                return plugging

        return None

    def getMangadbExposures(self):
        """Returns a list of mangaDB.Exposure objects with all the exposures
        for this plate."""

        scienceExps = self.getScienceExposures()
        mangaExposures = []
        for exp in scienceExps:
            if exp.mangadbExposure is None or len(exp.mangadbExposure) == 0:
                raise TotoroExceptions.TotoroError(
                    'platedb exposure_no={0} has no mangadb counterpart'
                    .format(exp.exposure_no))
            else:
                mangaExposures.append(exp.mangadbExposure[0])

        return mangaExposures

    def getScienceExposures(self):
        """Returns a list of all plateDB.Exposure science exposures for
        the plate."""

        scienceExps = []

        plateDB = self.db.plateDB

        for plugging in self.pluggings:
            exposures = self.session.query(plateDB.Exposure).join(
                plateDB.Observation, plateDB.ExposureFlavor).filter(
                    plateDB.Observation.plugging_pk == plugging.pk).filter(
                        plateDB.ExposureFlavor.label == 'Science').all()
            scienceExps += exposures

        return scienceExps

    def getTotoroExposures(self, onlySets=False):
        """Returns a list of Totoro dbclasses.Exposure instances for this
        Plate insance."""

        totoroExposures = []
        for ss in self.sets:
            totoroExposures += ss.totoroExposures

        if onlySets or self.isMock:
            return totoroExposures

        # Some exposures not in sets may be missing.
        allExposures = self.getMangadbExposures()
        totoroExposures += [TotoroExposure(exp.pk, parent='mangadb')
                            for exp in allExposures if exp.set_pk is None]

        return totoroExposures

    def getValidSets(self, includeIncomplete=False):

        validSets = []
        for set in self.sets:
            quality = set.getQuality()[0]
            if quality in ['Good', 'Excellent', 'Override Good']:
                validSets.append(set)
            elif includeIncomplete and quality == 'Incomplete':
                validSets.append(set)

        return validSets

    def getValidExposures(self, **kwargs):
        """Returns all valid exposures, even if they belong to an incomplete
        or bad set."""

        validExposures = []

        for set in self.sets:
            for exp in set.totoroExposures:
                if exp.isValid(**kwargs)[0] is True:
                    validExposures.append(exp)

        return validExposures

    def getHARange(self, intersect=False, mjd=None, **kwargs):

        ha0 = -self.mlhalimit
        ha1 = self.mlhalimit

        haRange = np.array([ha0, ha1]) % 360.
        haRange[haRange > 180.] -= 360.

        if intersect is False:
            return haRange

        if mjd is None:
            mjd = int(np.round(time.Time.now().mjd))
            log.debug('Plate.getHARange: MJD set to {0:d}'.format(mjd))
        else:
            mjd = int(mjd)

        jdRange = observingPlan.getMJD(mjd)

        if jdRange is None:
            warnings.warn('no observing block found for MJD={0:d}. '
                          'Observing windows will not be contrained.'
                          .format(mjd), TotoroExceptions.NoObservingBlock)
            return haRange

        observingRangeLST = np.array(map(site.localSiderealTime, jdRange))
        observingRangeHA = (observingRangeLST * 15 - self.ra) % 360.

        haRange = utils.getIntervalIntersection(haRange, observingRangeHA)
        haRange[haRange > 180] -= 360.

        return haRange

    def getLSTRange(self, **kwargs):

        haRange = self.getHARange(**kwargs)

        ha0, ha1 = haRange

        lst0 = (ha0 + self.coords[0]) % 360. / 15
        lst1 = (ha1 + self.coords[0]) % 360. / 15

        LSTRange = np.array([lst0, lst1])

        return LSTRange

    def getUTVisibilityWindow(self, **kwargs):
        return self.getUTRange(**kwargs)

    def getUTRange(self, mjd=None, returnType='str', **kwargs):

        lst0, lst1 = self.getLSTRange(**kwargs)

        if mjd is None:
            mjd = int(np.round(time.Time.now().mjd))

        date = time.Time(mjd, format='mjd', scale='tai')
        date0, date1 = site.localSiderealTimeToDate([lst0, lst1], date=date)

        # Sanity check
        if date0.jd > date1.jd:
            raise ValueError('date0 is greater than date1')

        if returnType == 'str':
            return ('{0:%H:%M}'.format(date0.datetime),
                    '{0:%H:%M}'.format(date1.datetime))
        elif returnType == 'datetime':
            return (date0.datetime, date1.datetime)
        else:
            return (date0, date1)

    def isVisible(self, LST0, LST1, minLength=None):
        """Returns True if the plate is visible in a range of LST."""

        if minLength is None:
            minLength = config['exposure']['exposureTime']

        lstIntersectionLength = utils.getIntervalIntersectionLength(
            self.getLSTRange(), (LST0, LST1), wrapAt=24.)

        secIntersectionLength = lstIntersectionLength * 3600.

        return True if secIntersectionLength >= minLength else False

    def isSpecial(self):
        """Returns True if a plate is all_sky or commissioning."""

        if self.mangadbPlate is None:
            return False
        if (self.mangadbPlate.all_sky_plate is True or
                self.mangadbPlate.commissioning_plate is True):
            return True
        return False

    def isNeverObserve(self):
        """Returns True if the plate is marked as neverobserve."""

        if self.mangadbPlate is None:
            return False
        if self.mangadbPlate.neverobserve is True:
            return True
        return False

    def addMockExposure(self, exposure=None, startTime=None, set=None,
                        expTime=None, silent=False, rearrange=True, **kwargs):
        """Creates a mock expusure in the best possible way."""

        ra, dec = self.coords

        if exposure is None:
            exposure = TotoroExposure.createMockExposure(
                startTime=startTime, expTime=expTime, ra=ra, dec=dec,
                dust=self.dust, **kwargs)

        validSet = plateUtils.getOptimalSet(self, exposure)

        if validSet is not None:
            # Gets a valid dither position for the exposure in this set.
            dither = plateUtils.getValidDither(validSet)
            exposure.ditherPosition = dither
            newSet = False

        else:
            # This is a new set. We give the exposure a random dither position.
            exposure.ditherPosition = 'N'
            validSet = TotoroSet(mock=True, **kwargs)
            newSet = True

        if exposure.isValid()[0] is False:
            if not silent:
                log.debug('mock exposure is invalid. Removing it.')
            return False

        if newSet:
            self.sets.append(validSet)
        validSet.totoroExposures.append(exposure)

        if rearrange:
            # We only run this rearrangement if the list of exposures in
            # incomplete sets is small
            nExpIncompleteSets = len(self.getExposuresInIncompleteSets())
            if nExpIncompleteSets <= 5:
                self.rearrangeSets(mode='optimal', scope='incomplete',
                                   silent=True)
            else:
                warnings.warn('plate={0}: skipping incomplete set '
                              'rearrangement because plate has > 5 exposures '
                              'in incomplete sets'.format(self.plate_id),
                              TotoroExceptions.TotoroUserWarning)

        return exposure

    def getIncompleteSet(self, startTime, expTime):
        """Returns incomplete sets that are valid for an exposure starting at
        JD=startTime."""

        LST0 = site.localSiderialTime(startTime)
        LST1 = (LST0 + expTime / 60.) % 24.

        incompleteSets = [set for set in self.sets if not set.complete]

        if len(incompleteSets) == 0:
            return None

        for set in incompleteSets:

            lstRange = set.getLSTRange()
            intersectionLength = utils.getIntervalIntersectionLength(
                (LST0, LST1), lstRange, wrapAt=24)

            if intersectionLength * 3600 > expTime:
                return set

        return None

    def getExposuresInIncompleteSets(self):
        """Returns a list of exposures in incomplete sets."""

        incompleteSetExposures = []

        incompleteSets = [set for set in self.sets if not set.complete]
        for ss in incompleteSets:
            incompleteSetExposures += ss.totoroExposures

        return incompleteSetExposures

    def getLastExposure(self):
        """Returns the last exposure taken."""

        exposures = self.getValidExposures()

        startTime = [exp.start_time for exp in exposures]
        order = np.argsort(startTime)

        return exposures[order[-1]]

    def getLastSet(self):
        """Returns the set containing the last exposure."""

        lastExposure = self.getLastExposure()

        for ss in self.sets:
            if lastExposure in ss.totoroExposures:
                return ss

    @property
    def priority(self):
        if not self.isMock and self._priority is None:
            return self.plate_pointings[0].priority
        else:
            if self._priority is None:
                return config['defaultPriority']
            else:
                return self._priority

    @priority.setter
    def priority(self, value):
        if not self.isMock:
            with self.session.begin():
                self.plate_pointings[0].priority = value
        else:
            self._priority = value

    @property
    def isPlugged(self):
        try:
            self.getActiveCartNumber()
            return True
        except:
            return False

    def getAPOcomplete(self, **kwargs):
        """Retuns the APOcomplete dictionary for this plate."""
        return utils.getAPOcomplete(self, **kwargs)

    def createAPOcompleteFile(self, path=None):
        """Creates an apocomp file for the current plate."""

        return utils.createAPOcompleteFile(self.getAPOcomplete(), path=path)

    @property
    def manga_tileid(self):
        """Returns manga_tileid for this plate."""
        if self._manga_tileid is not None:
            return self._manga_tileid

        if self.mangadbPlate is None:
            return None
        else:
            self._manga_tileid = self.mangadbPlate.manga_tileid
            return self._manga_tileid

    @manga_tileid.setter
    def manga_tileid(self, value):
        """Sets manga_tileid for this plate. Not saved to the DB."""
        self._manga_tileid = value

    def getMangaTileID(self):
        """Returns manga_tiled. Backwards compatibility."""
        return self.manga_tileid

    def getAltitude(self, LST=None):
        """Returns the altitude of the plate at a certain LST."""

        if LST is None:
            LST = site.localSiderealTime()

        HA = (LST * 15. - self.ra) % 360.

        sinAlt = (np.sin(np.deg2rad(self.dec)) *
                  np.sin(np.deg2rad(site.latitude)) +
                  np.cos(np.deg2rad(self.dec)) *
                  np.cos(np.deg2rad(site.latitude)) *
                  np.cos(np.deg2rad(HA)))

        return np.rad2deg(np.arcsin(sinAlt))

    def getLSTRangeAboveAltitude(self, altitude):
        """Returns the LST range in which the plate is above ``altitude`` [deg]
        """

        altRad = np.deg2rad(altitude)
        latRad = np.deg2rad(site.latitude)
        decRad = np.deg2rad(self.dec)

        xx = ((np.sin(altRad) - np.sin(latRad) * np.sin(decRad)) /
              (np.cos(latRad) * np.cos(decRad)))

        if xx > 1 or xx < -1:
            return np.array([0, 0])

        HA = np.rad2deg(np.arccos(xx))

        LSTRange = [(-HA + self.ra) % 360. / 15., (HA + self.ra) % 360. / 15.]

        return np.array(LSTRange)

    def getNumberPermutations(self):
        """Returns the number of permutations for a brute-force set
        rearrangement."""

        validExposures = self.getValidExposures()
        ditherPositions = [exp.ditherPosition for exp in validExposures]
        return plateUtils.getNumberPermutations(ditherPositions)

    def getPermutations(self):
        """Returns the permutations for a brute-force set rearrangement."""

        validExposures = self.getValidExposures()
        ditherPositions = [exp.ditherPosition for exp in validExposures]
        return plateUtils.calculatePermutations(ditherPositions)

    def getLocation(self):
        """Returns the location label of the plate."""

        if self.location is None:
            return None
        return self.location.label

    def getOrphaned(self, includeUnplugged=True, useMock=True):
        """Returns orphaned exposures in sets. Exposures must be valid
        and belong to a valid set. If `useMock=False`, only real sets will
        be considered. If `includeUnplugged=True`, orphaned exposures in
        unplugged sets will be included."""

        orphaned = []
        validStatuses = ['Incomplete']
        if includeUnplugged:
            validStatuses.append('Unplugged')

        for set in self.sets:
            # Creates a mock set with the appropriate exposures
            if not useMock:
                if set.isMock:
                    continue
                mockSet = TotoroSet.fromExposures(
                    [exp for exp in set.totoroExposures if not exp.isMock])
            else:
                mockSet = set

            # Checks if set is incomplete. If so, the exposures are orphaned.
            if mockSet.getQuality()[0] in validStatuses:
                orphaned += mockSet.totoroExposures

        return orphaned

    def getMockExposures(self):
        """Returns mock exposues in this plate."""

        result = []
        for ss in self.sets:
            for exp in ss.totoroExposures:
                if exp.isMock:
                    result.append(exp)

        return result

    def hasIncompleteSets(self):
        """Returns True if the plate has at least one incomplete set."""

        for ss in self.sets:
            if ss.getStatus()[0] == 'Incomplete':
                return True

        return False

    @property
    def drilled(self):
        """Property to record if a plate has already been drilled."""
        if self._drilled is not None:
            return self._drilled
        else:
            if self.plate_id is not None:
                return True
            else:
                return False

    @drilled.setter
    def drilled(self, value):
        self._drilled = value

    @property
    def inFootprint(self):
        """Returns True if the plate is in the MaNGA footprint.

        Caches the result.
        """

        if isinstance(self._inFootprint, bool):
            return self._inFootprint
        else:
            if self == getPlatesInFootprint(self):
                self._inFootprint = True
                return True
            else:
                self._inFootprint = False
                return False

    @inFootprint.setter
    def inFootprint(self, value):
        """Manually sets if a plate is in the footprint."""

        assert isinstance(value, bool)
        self._inFootprint = value

    @inFootprint.deleter
    def inFootprint(self):
        """Removes the caching for inFootprint."""

        self._inFootprint = None

    @property
    def dateAtAPO(self):
        return self._dateAtAPO

    @dateAtAPO.setter
    def dateAtAPO(self, value):
        assert value >= 0, 'value must be a JD.'
        self._dateAtAPO = value

    @property
    def dust(self):
        """Returns the dust value."""

        if self._dust is not None:
            return self._dust

        if dustMap is None:
            return None
        else:
            self._dust = dustMap(self.ra, self.dec)
            return self._dust

    @property
    def mlhalimit(self):
        """Returns the HA limit."""

        if self._mlhalimit is None:
            self._mlhalimit = utils.mlhalimit(self.dec)

        return self._mlhalimit


from py4ai.analytics.ml.wrapper.sklearn.transformer import (
    SelectByMutualInfo,
    SelectByFtest,
)
from sklearn.feature_selection import (
    mutual_info_classif,
    mutual_info_regression,
    SelectFpr,
)
from py4ai.data.model.ml import PandasDataset
import pandas as pd
import unittest
import numpy as np
from itertools import compress

features = ["f_1", "f_2", "f_3"]
label = "label"

X_c = [[0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 1, 1], [0, 1, 0], [0, 1, 1]]
y_c = [1, 0, 1, 1, 1, 0]
df_c = PandasDataset(
    features=pd.DataFrame(data=X_c, columns=features),
    labels=pd.Series(data=y_c, name=label),
)

X_r = np.random.rand(100, 3)
y_r = X_r[:, 0] + np.sin(6 * np.pi * X_r[:, 1]) + 0.1 * np.random.randn(100)
df_r = PandasDataset(
    features=pd.DataFrame(data=X_r, columns=features),
    labels=pd.Series(data=y_r, name=label),
)

class TestSelectByMutualInfo(unittest.TestCase):
    def setUp(self) -> None:
        self.estimator_classif = SelectByMutualInfo("Classification")
        self.estimator_regression = SelectByMutualInfo("Regression")

    def test_train(self):
        mi_c = mutual_info_classif(
            X_c,
            y_c,
            discrete_features=self.estimator_classif.discrete_features,
            n_neighbors=self.estimator_classif.n_neighbors,
            random_state=42,
        )

        mi_c /= np.max(mi_c) * 1.0
        to_keep = [
            i for i, k in zip(features, mi_c) if k > self.estimator_classif.mi_thresh
        ]
        feature_selector_c = self.estimator_classif.train(df_c)
        self.assertEqual(sorted(feature_selector_c.to_keep), sorted(to_keep))

        mi_r = mutual_info_regression(
            X_r,
            y_r,
            discrete_features=self.estimator_regression.discrete_features,
            n_neighbors=self.estimator_regression.n_neighbors,
            random_state=42,
        )

        mi_r /= np.max(mi_r) * 1.0
        to_keep = [
            i for i, k in zip(features, mi_r) if k > self.estimator_regression.mi_thresh
        ]
        feature_selector_r = self.estimator_regression.train(df_r)

        self.assertEqual(sorted(feature_selector_r.to_keep), sorted(to_keep))

class TestSelectByFtest(unittest.TestCase):
    def setUp(self) -> None:
        self.estimator_classif = SelectByFtest("Classification", alpha=0.5)
        self.estimator_regression = SelectByFtest("Regression", alpha=0.5)

    def test_train(self):
        sfpr_c = SelectFpr(
            score_func=lambda x, y: self.estimator_classif.func(x, y),
            alpha=self.estimator_classif.alpha,
        ).fit(X_c, y_c)

        to_keep_c = list(compress(features, sfpr_c.get_support()))
        feature_selector_c = self.estimator_classif.train(df_c)

        self.assertEqual(sorted(feature_selector_c.to_keep), sorted(to_keep_c))

        sfpr_r = SelectFpr(
            score_func=lambda x, y: self.estimator_regression.func(x, y),
            alpha=self.estimator_regression.alpha,
        ).fit(X_r, y_r)
        to_keep_r = list(compress(features, sfpr_r.get_support()))
        feature_selector_r = self.estimator_regression.train(df_r)
        self.assertEqual(sorted(feature_selector_r.to_keep), sorted(to_keep_r))

if __name__ == "__main__":
    unittest.main()


import gi
gi.require_version('Gimp', '3.0')
from gi.repository import Gimp
gi.require_version('GimpUi', '3.0')
from gi.repository import GimpUi
from gi.repository import GObject
from gi.repository import GLib
from gi.repository import Gio
import time
import sys

def N_(message): return message
def _(message): return GLib.dgettext(None, message)

def blobipy(procedure, run_mode, image, n_drawables, drawables, args, data):
    config = procedure.create_config()
    config.begin_run(image, run_mode, args)

    if run_mode == Gimp.RunMode.INTERACTIVE:
        GimpUi.init('python-fu-blobipy')
        dialog = GimpUi.ProcedureDialog(procedure=procedure, config=config)
        dialog.fill(None)
        if not dialog.run():
            dialog.destroy()
            config.end_run(Gimp.PDBStatusType.CANCEL)
            return procedure.new_return_values(Gimp.PDBStatusType.CANCEL, GLib.Error())
        else:
            dialog.destroy()

    color      = config.get_property('color')
    blur       = config.get_property('blur')

    Gimp.context_push()
    image.undo_group_start()

    # Alpha to selection. pdb.gimp_selection_layer_alpha(layer)
    # is deprecated and replaced by gimp-image-select-item.
    # The arguments are image (GimpImage), operation (GimpChannelOps)
    # and item (GimpItem).
    Gimp.get_pdb().run_procedure('gimp-image-select-item', [
        GObject.Value(Gimp.Image, image),
        # Gimp.ChannelOps.REPLACE is an enum
        GObject.Value(GObject.TYPE_INT, Gimp.ChannelOps.REPLACE),
        GObject.Value(Gimp.Drawable, drawables[0])
    ])

    # Invert the selection:
    Gimp.get_pdb().run_procedure('gimp-selection-invert', [
        GObject.Value(Gimp.Image, image),
    ])

    # Make a dropshadow from the inverted selection
    Gimp.get_pdb().run_procedure('script-fu-drop-shadow', [
        GObject.Value(Gimp.RunMode, Gimp.RunMode.NONINTERACTIVE),
        GObject.Value(Gimp.Image, image),
        GObject.Value(Gimp.Drawable, drawables[0]),
        GObject.Value(GObject.TYPE_INT, -3),      # offset X
        GObject.Value(GObject.TYPE_INT, -3),      # offset Y
        GObject.Value(GObject.TYPE_INT, blur),    # blur in pixels
        # Should color be cast into some GObject type? What type?
        # There is no GObject.TYPE_COLOR
        color,
        GObject.Value(GObject.TYPE_INT, 80), # opacity
        GObject.Value(GObject.TYPE_BOOLEAN, False) # allow resizing
    ])

    # Finish and clean up
    Gimp.Selection.none(image)
    image.undo_group_end()
    Gimp.context_pop()

    config.end_run(Gimp.PDBStatusType.SUCCESS)

    return procedure.new_return_values(Gimp.PDBStatusType.SUCCESS, GLib.Error())

class BlobiPy(Gimp.PlugIn):
    ## Parameters ##
    __gproperties__ = {
        "blur": (int,
                 _("Blur amout"),
                 _("Blur amount"),
                 0, 50, 7,
                 GObject.ParamFlags.READWRITE),
    }

    ## GimpPlugIn virtual methods ##
    def do_set_i18n(self, procname):
        return True, 'gimp30-python', None

    def do_query_procedures(self):
        return [ 'python-fu-blobipy' ]

    def do_create_procedure(self, name):
        # Workaround copied from foggify.py until GIMP can handle color args
        # in the regular __gproperties__ list
        black = Gimp.RGB()
        black.set(0, 0, 0)
        color = GObject.Property(type =Gimp.RGB, default=black,
                                 nick =_("_Shadow color"),
                                 blurb=_("Shadow color"))

        procedure = Gimp.ImageProcedure.new(self, name,
                                            Gimp.PDBProcType.PLUGIN,
                                            blobipy, None)
        procedure.set_image_types("RGB*, GRAY*");
        procedure.set_sensitivity_mask (Gimp.ProcedureSensitivityMask.DRAWABLE |
                                        Gimp.ProcedureSensitivityMask.DRAWABLES)
        procedure.set_documentation(_("Create a 3-D effect"),
                                    _("Create a blobby 3-D effect "
                                      "using an inverse drop-shadow."),
                                    name)
        procedure.set_menu_label(_("_BlobiPy..."))
        procedure.set_attribution("Akkana Peck", "Akkana Peck", "2009,2022")
        procedure.add_menu_path ("<Image>/Filters/Light and Shadow")

        procedure.add_argument_from_property(self, "blur")
        procedure.add_argument_from_property(self, "color")
        return procedure

Gimp.main(BlobiPy.__gtype__, sys.argv)


import torch
from torch import nn
import math
from models.stylegan2.model import ConvLayer

class Mask(nn.Module):

    def __init__(self, channel_multiplier=2, size=256):
        super(Mask, self).__init__()

        channels = {
            8: 512,
            16: 512,
            32: 512,
            64: 256 * channel_multiplier,
            128: 128 * channel_multiplier,
            256: 64 * channel_multiplier,
        }

        self.log_size = (int(math.log(size, 2)) - 1) * 2
        self.convs = nn.ModuleList()

        for i in range(2, self.log_size):
            idx = i // 2
            s = 2 ** (idx + 2)
            self.convs.append(ConvLayer(channels[s], 32, 3))
            scale = size // s
            self.convs.append(nn.Upsample(scale_factor=scale))

        self.to_mask = nn.Sequential(*[
            ConvLayer(32 * 12, 128, 3),
            ConvLayer(128, 1, 3, activate=False),
            nn.Sigmoid()
        ])

    def forward(self, feats):
        feats = feats[2:-4]
        feats_rs = []

        for i, (conv, up) in enumerate(zip(self.convs[::2], self.convs[1::2])):
            feats_rs.append(up(conv(feats[i])))

        mask = self.to_mask(torch.cat(feats_rs, dim=1))
        return mask


import logging
import os
import time
import traceback
from datetime import datetime
from multiprocessing import Process

from automonkey.config import DefaultConfig
from .adb import AdbTool
from .exception import DeviceNotConnectedException, InstallAppException, CheckScreenLockedFailed
from .logcat import LogCat
from .tcloud_update import TCloud
from .utils import Utils

logger = logging.getLogger(__name__)

"""
#  MonkeyRunner ， 多进程
"""

class MonkeyRunner(Process):

    def __init__(self, queue, lock, monkey):
        super(MonkeyRunner, self).__init__()
        self.queue = queue
        self.monkey = monkey
        self.lock = lock
        self.daemon = True
        self.logcat = None
        self.adb_tool = None
        self.tcloud = None
        self.anr = 0
        self.crash = 0
        self.local_monkey_path = None
        self.local_logcat_path = None
        self.local_crash_path = None
        self.process = 0

    def run(self):
        try:
            self.init_tools()

            # 开始测试
            self.tcloud.on_task_begin()

            # 测试准备
            self.setup()

            # 测试内容
            self.run_monkey()

            # 测试结束
            self.teardown()

        except DeviceNotConnectedException as d_e:
            logger.error(d_e)
            self.monkey.result.on_device_connect_failed()
            self.tcloud.on_device_connect(False)
        except CheckScreenLockedFailed as c_e:
            logger.error(c_e)
            self.monkey.result.on_check_screen_lock_failed()
            self.tcloud.on_screen_lock(False)
        except InstallAppException as i_e:
            logger.error(i_e)
            self.monkey.result.on_app_install_failed()
            self.tcloud.on_setup_install_app(False)
        except CheckScreenLockedFailed as s_e:
            logger.error(s_e)
            self.monkey.result.on_check_screen_lock_failed()
            self.tcloud.on_screen_lock(False)
        except Exception as e:
            logger.error(e)
            traceback.print_exc()
        finally:
            logger.info('{} 结束测试 设备 {}'.format(self.pid, self.monkey.device.device_id))
            # 测试结束
            self.teardown()
            self.publish_result()

    def run_monkey(self):
        try:
            # 回到桌面
            logger.info('({}) 回到桌面'.format(self.monkey.device.device_id))
            self.adb_tool.back_to_home()

            # 热启动 app
            logger.info('({}) 尝试热启动 app'.format(self.monkey.device.device_id))
            self.monkey.result.start_app_status = self.adb_tool.start_activity(self.monkey.config.package_name,
                                                                               self.monkey.config.default_app_activity)
            self.tcloud.on_start_app(self.monkey.result.start_app_status)

            # 登陆 app
            self.monkey.result.login_app_status = self.try_to_login_app()
            self.tcloud.on_login_app(self.monkey.result.login_app_status)

            monkey_progress = self.run_monkey_cmd(self.monkey.config.run_time)

            time_begin = datetime.now()

            current_anr = 0
            current_crash = 0
            running_status = 1
            running_error_reason = ''
            cancel_flag = False

            while True:

                time_temp = (datetime.now() - time_begin).seconds

                # 检测 monkey_progress 是否
                if monkey_progress.poll() is not None:
                    logger.warning('[{}] monkey stoped early !'.format(self.monkey.device.device_id))
                    logger.info('[{}] try to rerun the monkey!'.format(self.monkey.device.device_id))

                    mins = self.monkey.config.run_time - time_temp // 60

                    if mins <= 0:
                        current_anr, current_crash = self.on_process_crash_anr_changed(time_temp)
                        self.tcloud.on_anr_crash_changed(100, current_anr, current_crash)
                        break
                    monkey_progress = self.run_monkey_cmd(mins)

                # 检测是否 进行了 中断操作
                cancel_flag = self.on_user_cancel()

                if cancel_flag:
                    break

                # 检测 设备连接和锁屏状态
                if isinstance(self.monkey.device.connect(), DeviceNotConnectedException):
                    running_error_reason = 'device {} dis connect'.format(self.monkey.device.device_id)
                    running_status = 2
                    self.tcloud.on_device_disconnect_on_running()
                    break
                self.adb_tool.check_screen_locked()

                # process crash anr 改变
                self.on_process_crash_anr_changed(time_temp)

                # 当 process == 100 , break
                if self.process == 100:
                    break

                time.sleep(5)

            time.sleep(30)  # 此处的需要等待 log 写入

            monkey_logs = []

            # 等待 monkey 结束
            while monkey_progress.poll() is None:
                logger.info('waiting for monkey end...')
                time_temp = (datetime.now() - time_begin).seconds // 60
                if time_temp > 1:
                    # 这里暂时不获取 monkey log
                    # monkey_logs = monkey_progress.stdout.readlines()
                    if monkey_progress.poll() is None:
                        monkey_progress.kill()
                    break
                time.sleep(10)

            # user cancel here
            if cancel_flag:
                self.tcloud.on_user_cancel_stask_success()
            else:
                self.tcloud.on_running_status(running_status, running_error_reason)

            self.monkey.result.log_path = self.logcat.get_logcat_log(monkey_logs)

            activity_all, activity_tested, activity_info = self.get_activity_infos()
            self.monkey.result.activity_info = activity_info
            # update task end info
            self.tcloud.on_task_end(process=self.process, activity_count=len(activity_all),
                                    activity_tested_count=len(activity_tested),
                                    activity_all=str(activity_all), activity_tested=str(activity_tested),
                                    anr_count=current_anr, crash_count=current_crash, crash_rate=0,
                                    exception_count=0, exception_run_time=0, run_time=self.monkey.config.run_time)

            # 重复上传，删除
            # self.analysis_upload_crash_logs()
            self.get_gen_bug_report()
            self.upload_other_report()
            time.sleep(10)

        except Exception as e:
            logger.error(e)
            logger.error(traceback.format_exc())
        finally:
            self.teardown()

    def on_user_cancel(self):
        cancel_status = self.tcloud.get_monkey_cancel_status(self.monkey.task_id)
        logger.info(cancel_status)
        if cancel_status in ['0', 0]:
            logger.warning('[{}] here find the cancel from the tcloud, stop the monkey now!'.
                           format(self.monkey.device.device_id))
            running_error_reason = 'cancel by user!'
            # on user cancel
            self.tcloud.on_user_cancel_task()
            cancel_flag = True
        else:
            cancel_flag = False
        return cancel_flag

    def on_process_crash_anr_changed(self, time_temp):
        logger.info('({}) time now ... {}'.format(self.monkey.device.device_id, time_temp))
        self.process = int(time_temp / (self.monkey.config.run_time * 60.00) * 100.00)
        logger.info('({}) process now : {}'.format(self.monkey.device.device_id, self.process))

        if self.process <= 0:
            self.process = 0

        if self.process >= 100:
            self.process = 100

        current_activity = self.adb_tool.get_current_activity()
        current_battery_level = self.adb_tool.get_battery_level()
        current_anr, current_crash = self.logcat.get_anr_crash_count()

        logger.info(
            '''current information <{}> current activity : {} ; current battery : {}; anr count : {} ;'''
            '''crash count : {}'''.format(datetime.now().strftime('%Y.%m.%d %H:%M:%S'), current_activity,
                                          current_battery_level, current_anr, current_crash))
        self.on_anr_crash_changed(current_anr, current_crash)
        # 这里防止 crash anr 变小，使用系统存储的 anr 和 crash
        current_crash = self.crash
        current_anr = self.anr

        self.tcloud.on_anr_crash_changed(self.process, current_anr, current_crash)
        return current_anr, current_crash

    def run_monkey_cmd(self, run_time=0):
        run_mode = DefaultConfig.MONKEY_MODE_KEY_MAP.get(int(self.monkey.config.run_mode))
        logcat_process = self.logcat.set_logcat(self.local_logcat_path)

        # 开始执行 monkey
        logger.info('({}) 开始运行 monkey 测试'.format(self.monkey.device.device_id))
        logger.info('({}) monkey mode : {}'.format(self.monkey.device.device_id, run_mode))

        command = 'shell CLASSPATH=/sdcard/monkey.jar:/sdcard/framework.jar exec app_process ' \
                  ' /system/bin tv.panda.test.monkey.Monkey -p {} {} --throttle 500 ' \
                  ' --output-directory /sdcard/MonkeyLog --running-minutes {} -v -v > {}'.format(
            self.monkey.config.package_name, run_mode, run_time, self.local_monkey_path)

        logger.info('({}) 开始运行 monkey 测试 [{} 分钟], 日志输出到 {}'.format(
            self.monkey.device.device_id, run_time, self.local_monkey_path))

        monkey_progress = self.adb_tool.run_monkey(command)
        return monkey_progress

    def get_activity_infos(self):
        activity_info = self.logcat.get_activity_test_info(show_in_cmd=True)
        activity_all = activity_info.get('TotalActivity')
        activity_tested = activity_info.get('TestedActivity')
        return activity_all, activity_tested, activity_info

    def analysis_upload_crash_logs(self):
        crash_logs = self.logcat.analysis_crash_anr_log(self.local_crash_path)
        self.tcloud.on_errorlog_upload(crash_logs)

    # 当 anr 或者 crash 改变的时候， 获取并上传 anr crash log，实时显示
    def on_anr_crash_changed(self, anr, crash):
        try:
            if self.anr != anr or self.crash != crash:
                if self.anr < anr or self.crash < crash:
                    logger.info('here anr or crash changed !')
                    self.logcat.get_crash_dump_log()
                    crash_logs = self.logcat.analysis_crash_anr_log(self.local_crash_temp_path, self.anr, self.crash)
                    self.anr = anr
                    self.crash = crash
                    self.tcloud.on_errorlog_upload(crash_logs)
                    return True
            return False
        except Exception as e:
            logger.error(e)
            logger.error(traceback.format_exc())

    # 获取并生成 bug report（adb bug report)
    def get_gen_bug_report(self):
        try:
            self.logcat.get_bug_report_log()
            self.logcat.generate_bug_report()
            self.logcat.upload_bug_report_log()

            now = datetime.now().strftime("%Y-%m-%d")
            build_number = os.environ.get('BUILD_NUMBER')
            report_url = '{}/monkey/{}/{}/logcat/{}/bug_report_out/index.html'.format(DefaultConfig.OSS_MONKEY_URL,
                                                                                now, build_number, self.pid)
            report_type = 1
            self.tcloud.on_report_upload(report_url, report_type)
        except Exception as e:
            logger.error(e)
            logger.error(traceback.format_exc())

    def upload_other_report(self):
        try:
            now = datetime.now().strftime("%Y-%m-%d")
            build_number = os.environ.get('BUILD_NUMBER')
            # monkey.log
            report_url_pre = '{}/monkey/{}/{}/logcat/{}'.format(DefaultConfig.OSS_MONKEY_URL, now, build_number,
                                                                self.pid)

            self.tcloud.on_report_upload(report_url='{}/monkey.log'.format(report_url_pre), report_type=2)

            if self.anr > 0 or self.crash > 0:
                self.tcloud.on_report_upload(report_url='{}/monkey/MonkeyLog/crash-dump.log'.format(report_url_pre),
                                             report_type=3)
        except Exception as e:
            logger.error(e)
            logger.error(traceback.format_exc())

    def publish_result(self):
        self.lock.acquire()
        self.queue.put(self.monkey)
        self.lock.release()

    def init_tools(self):
        self.adb_tool = AdbTool(self.monkey.device.device_id)
        self.tcloud = TCloud(self.monkey.task_id, self.monkey.device.device_id, self.monkey.monkey_id,
                             self.monkey.tcloud_url)

        self.local_monkey_path = '{}/{}/{}'.format(DefaultConfig.LOCAL_LOGCAT_PATH, self.pid, 'monkey.log')
        self.local_logcat_path = '{}/{}/{}'.format(DefaultConfig.LOCAL_LOGCAT_PATH, self.pid, 'logcat.log')
        self.local_crash_path = '{}/{}/monkey/MonkeyLog/crash-dump.log'.format(DefaultConfig.LOCAL_LOGCAT_PATH,
                                                                               self.pid)
        self.local_crash_temp_path = '{}/{}/crash-dump.log'.format(DefaultConfig.LOCAL_LOGCAT_PATH, self.pid)

    def setup(self):
        try:
            logger.info('{} 开始测试 设备 {}'.format(self.pid, self.monkey.device.device_id))

            # 清除 重建 logs
            self.clear_local_logs()
            self.creat_local_log_path()

            # 连接设备
            self.monkey.result.device_connect_result = self.monkey.device.connect()

            if isinstance(self.monkey.result.device_connect_result, DeviceNotConnectedException):
                raise DeviceNotConnectedException

            self.tcloud.on_device_connect(True)
            # self.adb_tool.u2helper.connect()

            # 锁定机器
            self.tcloud.using_monkey_device(self.monkey.device.device_id)

            # 开启日志
            self.logcat = LogCat(self.monkey.device.device_id, self.pid)

            # 测试开始时间
            self.monkey.result.on_case_begin()

            # 将 monkey.jar 和 framework.jar push 到 /sdcard
            self.adb_tool.push_file('./tools/monkey.jar', '/sdcard/')
            self.adb_tool.push_file('./tools/framework.jar', '/sdcard/')
            self.adb_tool.push_file('./tools/max.config', '/sdcard/')

            if self.monkey.config.install_app_required:

                # 卸载
                self.monkey.result.setup_uninstall_result = self.adb_tool.uninstall_package(
                    self.monkey.config.package_name)
                self.tcloud.on_setup_uninstall_app(self.monkey.result.setup_uninstall_result)

                # 安装 package
                self.monkey.result.setup_install_result = self.adb_tool.install_package(
                    self.monkey.config.local_package_path, self.monkey.config.package_name)
            else:
                self.monkey.result.setup_install_result = True

            self.tcloud.on_setup_install_app(self.monkey.result.setup_install_result)
            if not self.monkey.result.setup_install_result:
                raise InstallAppException

            # 获取 app 版本
            self.monkey.result.app_version = self.adb_tool.get_package_version(self.monkey.config.package_name)

            self.tcloud.on_get_app_version(self.monkey.result.app_version)

            # 检查 设备锁屏状态
            self.monkey.result.check_screen_locked = self.adb_tool.check_screen_locked()

            self.tcloud.on_screen_lock(self.monkey.result.check_screen_locked)

            if not self.monkey.result.check_screen_locked:
                raise CheckScreenLockedFailed

            self.clear_log_on_device()
            time.sleep(5)

        except Exception as e:
            logger.error(e)
            traceback.print_exc()
            raise e

    def teardown(self):
        try:

            logger.info('{} 结束测试 设备 {}'.format(self.pid, self.monkey.device.device_id))
            # 测试结束时间
            self.monkey.result.on_case_end()

            # 卸载 package
            self.monkey.result.teardown_uninstall_result = self.adb_tool.uninstall_package(
                self.monkey.config.package_name)

            self.tcloud.on_teardown_uninstall_app(self.monkey.result.teardown_uninstall_result)

            self.tcloud.release_monkey_device(self.monkey.device.device_id)

            # 断开连接设备
            self.monkey.device.disconnect()
        except Exception as e:
            logger.error(e)
            traceback.print_exc()
        finally:
            self.tcloud.release_monkey_device(self.monkey.device.device_id)

    def clear_log_on_device(self):
        # 移除 设备中原有的 log /sdcard 下的log
        self.adb_tool.remove_file('/sdcard/crash-dump.log')
        self.adb_tool.remove_file('/sdcard/oom-traces.log')
        self.adb_tool.remove_file('/sdcard/MonkeyLog')
        self.logcat.reset_bug_report_log()

    def clear_local_logs(self):
        # 移除本地logcat 目录
        logger.info('移除 logcat 目录')
        Utils.command_execute('rm -rf ./logcat')

    def creat_local_log_path(self):
        logger.info('初始化 log 目录')

        Utils.command_execute('mkdir -p logcat')
        Utils.command_execute('mkdir -p logcat/{}'.format(self.pid))

    def try_to_unlock_screen(self):
        logger.info('({}) 尝试 解锁屏幕'.format(self.monkey.device.device_id))
        # TODO

    def try_to_login_app(self):
        logger.info('({}) 尝试 登陆 app {}'.format(self.monkey.device.device_id, self.monkey.config.package_name))
        # TODO
        try:
            if self.monkey.config.login_required:
                logger.info('({}) 使用 [{} ,{}] 尝试登陆'.format(self.monkey.device.device_id,
                                                           self.monkey.config.login_username,
                                                           self.monkey.config.login_password))
                return True
            else:
                logger.info('({}) 不需要登陆'.format(self.monkey.device.device_id))
                return True
        except Exception as e:
            logger.error(e)
            logger.error(traceback.format_exc())
            return False


import sys
from PyQt5.QtWidgets import QMainWindow, QApplication, QLabel, QToolBar, QAction, QStatusBar, QVBoxLayout, QPushButton, QDialog, QWidget

# Class for creating another window in the app
class AnotherWindow(QWidget):
    def __init__(self):
        super().__init__()

        # Creates a layout for the label widget
        layout = QVBoxLayout()
        # Creates the label
        self.label = QLabel('Another Window')
        #Adds the label to the widget
        layout.addWidget(self.label)
        #sets the layout of the window
        self.setLayout(layout)

class MainWindow(QMainWindow):

    def __init__(self):
        super(MainWindow, self).__init__()

        self.w = None

        # Sets the window title
        self.setWindowTitle("PyQT5 MultiWindow Example")

        # Creates a button
        self.button = QPushButton('Push this button for a new window')
        # Connects the button to the callback
        self.button.clicked.connect(self.show_new_window)
        # Sets the button as the central widget
        self.setCentralWidget(self.button)

    # Callback function for the button
    def show_new_window(self, checked):
        # If the window is not open, it creates a new window
        if self.w is None:
            self.w = AnotherWindow()
        # shows the window
        self.w.show()

def createWindow():

    app = QApplication(sys.argv)

    window = MainWindow()
    window.show()

    # Start the event loop.
    app.exec()

# Press the green button in the gutter to run the script.
if __name__ == '__main__':
    createWindow()

PyQT5 Projects/PyQT5- Dialogs/PyQT5 - MultiWIndow Basic.py


from collections import  deque

def solution(i):
    if len(i) % 2 == 0:
        return -1
    elif len(i) == 1:
        return 0

    dq = deque(i)

    all_match = True
    possible_answer = (len(i)-1)//2
    for n in range(0, possible_answer):
        all_match = dq.popleft() == dq.pop()
        if not all_match:
            break

    if all_match:
        return possible_answer
    else:
        return -1

if __name__ == '__main__':
    inputs = ["x","racecar","madam","aaabbb","xxaxx","cifXfic"]

    for i in inputs:
        print(i,"->",solution(i))


from pathlib import Path

from . import fs
from ..sys import fs as sys_fs
from ..utils import link_config

LETSENCRYPT_DIR = '/etc/letsencrypt'

def bootstrap(ctx):
    """Bootstraps Certbot for the project."""
    fs.create_dir(ctx, ctx.paths.remote.project.state.certbot)
    project_name = ctx.project.name

    # Prepare directories in /etc/letsencrypt/
    ctx.sudo('certbot', warn=True)

    # Link a deploy hook, to allow project user access to it.
    link_config(
        ctx,
        title='certbot hook',
        name_local=project_name + '-certbot-hook.sh',
        name_remote=project_name,
        dir_remote_confs=Path(f'{LETSENCRYPT_DIR}/renewal-hooks/deploy/')
    )

    get_certificate(ctx)

def get_certificate(ctx):
    """Get certificates from Certbot for HTTPS using webroot plugin.

    :param ctx:

    """
    project = ctx.project
    group = ctx.project.group
    domain = project.domain
    email = project.email or ''
    webroot = ctx.paths.remote.project.state.certbot

    if email:
        email = f'--email {email}'

    command = (
        f'certbot --agree-tos --no-eff-email {email} certonly '
        f'--webroot -d {domain} -w {webroot}')

    ctx.sudo(command)

    # Set access for
    for realm in ['archive', 'live']:
        sys_fs.setfacl(
            ctx,
            path=f'{LETSENCRYPT_DIR}/{realm}/',
            acl=f'g:{group}:--x', modify=True)

        sys_fs.setfacl(
            ctx,
            path=f'{LETSENCRYPT_DIR}/{realm}/{domain}/',
            acl=f'g:{group}:r-x', modify=True)

def dump(ctx, target_dir):
    """Dumps Certbot related stuff into a target directory."""

    sys_fs.gzip_dir(
        ctx,
        LETSENCRYPT_DIR,
        target_dir,
        do_sudo=True
    )

def restore(ctx, source_dir):
    """Restores Certbot related stuff from a source directory."""

    sys_fs.gzip_extract(
        ctx,
        archive=source_dir / 'certbot.tar.gz',
        target_dir=LETSENCRYPT_DIR,
        do_sudo=True
    )


import experiment.runtime.utilities.container_image_cache
import pytest

@pytest.fixture(params=[
    {'uri': 'http://domain/image:tag', 'schema': 'http://', 'domain': 'domain/', 'image': 'image', 'tag': ':tag'},
    {'uri': 'http://domain/image@tag', 'schema': 'http://', 'domain': 'domain/', 'image': 'image', 'tag': '@tag'},
    {'uri': 'http://domain/image@sha256:1', 'schema': 'http://', 'domain': 'domain/', 'image': 'image',
        'tag': '@sha256:1'},
    {'uri': 'http://a.b/c/image', 'schema': 'http://', 'domain': 'a.b/c/', 'image': 'image', 'tag': ''},
    {'uri': 'docker-pullable://a.b/c/image@sha256:abc', 'schema': 'docker-pullable://', 'domain': 'a.b/c/',
        'image': 'image', 'tag': '@sha256:abc'},
    {'uri': 'http://a.b/c/image@sha256:1', 'schema': 'http://', 'domain': 'a.b/c/', 'image': 'image', 'tag': '@sha256:1'},
    {'uri': 'domain/image@tag', 'schema': '', 'domain': 'domain/', 'image': 'image', 'tag': '@tag'},
    {'uri': 'image@tag', 'schema': '', 'domain': '', 'image': 'image', 'tag': '@tag'},
    {'uri': 'image', 'schema': '', 'domain': '', 'image': 'image', 'tag': ''},
])
def image_breakdown(request):
    return request.param

def test_partition_image_uri(image_breakdown):
    schema, domain, image, tag = experiment.runtime.utilities.container_image_cache.partition_image_uri(image_breakdown['uri'])

    assert schema == image_breakdown['schema']
    assert domain == image_breakdown['domain']
    assert image == image_breakdown['image']
    assert tag == image_breakdown['tag']

    # VV: Ensure that putting everything back together generates the original URI
    assert ''.join((schema, domain, image, tag)) == image_breakdown['uri']

def test_expand_image_uri_tag(image_breakdown):
    img = experiment.runtime.utilities.container_image_cache.expand_image_uri(image_breakdown['uri'])
    schema, domain, image, tag = experiment.runtime.utilities.container_image_cache.partition_image_uri(img)
    assert tag == image_breakdown['tag'] or ':latest'


# -*- coding: utf-8 -*-
"""
Created on Mon Jun  8 14:20:43 2020

@author: Administrator
"""

from flask import Flask,request,jsonify
import time
import datetime

from . import api
from app.model import get_db

@api.route('/behavior/proportion_results', methods=['POST'])
def proportion_results():
    '''
    获
    '''
    data = request.get_json()

    past_por_n = data['past_pro_n']

    db = get_db()['invitationCat']

    t_now = datetime.datetime.today()
    t_at_zero = t_now.replace(hour=0,minute=0,second=0,microsecond=0)
    past_str = str(t_at_zero - datetime.timedelta(days = int(past_por_n)))
    pastn_time_array = time.strptime(past_str,'%Y-%m-%d %H:%M:%S')
    past_n_stamp =  int(time.mktime(pastn_time_array) * 1000)

    pastpron_user_plays = db.actions.find({
            'behavior': 0,
            'createdAt':{'$gte':float(past_n_stamp)}
            }).count()#过去n天观看

    pastpron_user_comments = db.comments.find({
            'status': '1',
            'createdAt':{'$gte':float(past_n_stamp)}
            }).count()#过去n天评论

    pastpron_user_danmakus = db.danmakus.find({
            'status': '1',
            'createdAt':{'$gte':float(past_n_stamp)}
            }).count()#过去n天弹幕

    pastpron_user_follows = db.follows.find({
            'type':{'$in':[1,2,3]},
            'createdAt':{'$gte':float(past_n_stamp)}
            }).count()#过去n天收藏

    pastpron_user_preferences = db.preferences.find({
            'state': 1,
            'createdAt':{'$gte':float(past_n_stamp)}
            }).count()#过去n天点赞

    pastpron_user_shares = db.actions.find({
            'behavior': 6,
            'createdAt':{'$gte':float(past_n_stamp)}
            }).count()#过去n天分享

    all_pro_nums = pastpron_user_plays + pastpron_user_comments + pastpron_user_danmakus + pastpron_user_follows+\
        pastpron_user_preferences+pastpron_user_shares

    return jsonify({
            'plays_pro_n':{float(pastpron_user_plays / all_pro_nums) * 100:'%'},
            'comments_pro_n':{float(pastpron_user_comments / all_pro_nums) * 100:'%'},
            'danmakus_pro_n':{float(pastpron_user_danmakus / all_pro_nums) * 100:'%'},
            'follows_pro_n':{float(pastpron_user_follows / all_pro_nums) * 100:'%'},
            'preferences_pro_n':{float(pastpron_user_preferences / all_pro_nums) * 100:'%'},
            'shares_pro_n':{float(pastpron_user_shares / all_pro_nums) * 100:'%'}
            })


import pygame

pygame.init()
SCREEN_WIDTH = 795
SCREEN_HEIGHT = 410
WHITE = (0, 0, 0)
Start_Screen = pygame.display.set_mode((SCREEN_WIDTH, SCREEN_HEIGHT))
pygame.display.set_caption('STRANDED')

# Images

background = pygame.image.load('Assets/Images/Background_Image.jpg')
background = pygame.transform.scale(background, (800, 409))

Start_Button_Image = pygame.image.load('Assets/Images/Start_Game_Button.png').convert_alpha()
Start_Button_Image = pygame.transform.scale(Start_Button_Image, (300, 300))

# Music

Start_Music = pygame.mixer.music.load('Assets/Sound/Start_Screen_Music.mp3')
pygame.mixer.music.play(-1)

#Button
class Button:
    def __init__(self, x, y, image):
        self.image = image
        self.rect = self.image.get_rect()
        self.rect.topleft = (x, y)
        self.clicked = False

    def draw(self):
        action = False
        # draw button on screen
        Start_Screen.blit(self.image, (self.rect.x, self.rect.y))

        pos = pygame.mouse.get_pos()

        def game_loop(self):
            if self.rect.collidepoint(pos):
                if pygame.mouse.get_pressed()[0] != 1 or self.clicked != False:
                    pass
                else:
                    self.clicked = True
                    action = True

            if pygame.mouse.get_pressed()[0] == 0:
                self.clicked = False

            return action

start_button = Button (250, 60, Start_Button_Image)

# Game Events

GamePlaying = True
while GamePlaying:

    for event in pygame.event.get():
        if event.type == pygame.QUIT:

            GamePlaying = False

    Start_Screen.blit(background, (0, 0))

    if start_button.draw():
        print('START')

    pygame.display.update()

pygame.quit()


import io

from opsconfig import menu
from opsconfig import nginxparser
from opsconfig import utils
from opsconfig.nginxConfigManipulator import NginxConfigManipulator

def run(stage="dev"):
    if stage == "dev":
        nginxConfigsDir = "tests/nginxconfig/"
    elif stage == "test":
        nginxConfigsDir = "/etc/nginx/sites-available/"
    else:
        nginxConfigsDir = "/etc/nginx/sites-available/"

    nginxConfigs = utils.getNginxConfigs(nginxConfigsDir)
    installers = []
    for nginxConfig in nginxConfigs:
        configFile = nginxConfig
        installer = NginxConfigManipulator(nginxConfigsDir + configFile)
        installers.append(installer)
    selection, infos = menu.installMenu(installers)

    services = []
    for i in range(len(infos)):
        installer = installers[i]
        service = [infos[i][s] for s in selection[i]]
        services.extend(service)
        installer.insertBlocks([s["index"] for s in service])
        if stage == "dev":
            nginxDump = nginxparser.dumps(installer.parsed)
            print(nginxDump)
        else:
            with io.open(installer.configFile, "w", encoding="utf-8") as config:
                nginxparser.dump(installer.parsed, config)
    return services


from pyqtgraph.Qt import QtCore, QtGui
import traceback
import sys
from multiprocessing import Process, Pool, Queue, Pipe
import time

class WorkerSignals(QtCore.QObject):
    '''
    Defines the signals available from a running worker thread.

    Supported signals are:

    finished
        No data

    error
        `tuple` (exctype, value, traceback.format_exc() )

    result
        `object` data returned from processing, anything

    progress
        `int` indicating % progress

    '''

    finished = QtCore.pyqtSignal()
    error = QtCore.pyqtSignal(tuple)
    result = QtCore.pyqtSignal(object)
    progress = QtCore.pyqtSignal(int)

class Worker(QtCore.QRunnable):
    '''
    Worker thread

    Inherits from QRunnable to handler worker thread setup, signals and wrap-up.

    :param callback: The function callback to run on this worker thread. Supplied args and
                     kwargs will be passed through to the runner.
    :type callback: function
    :param args: Arguments to pass to the callback function
    :param kwargs: Keywords to pass to the callback function

    '''

    def __init__(self, fn, *args, progress_fn=None, useMultiProcessing=False, verbose=False, **kwargs):
        super().__init__()

        # Store constructor arguments (re-used for processing)
        self.fn = fn
        self.useMultiProcessing = useMultiProcessing
        self.args = args
        self.kwargs = kwargs
        self.signals = WorkerSignals()
        self.verbose = verbose
        self.progress_fn = progress_fn

        process_pipe, main_pipe = Pipe()
        self.receiver = main_pipe
        self.sender = process_pipe

        # Add the callback to our kwargs
        # self.kwargs['progress_callback'] = self.signals.progress

    def progress_update(self, queue, progress):
        # queue.put(progress)
        # get_progress(queue)
        # self.signals.progress.emit(progress)
        if self.progress_fn is not None:
            self.progress_fn(progress)

    @QtCore.pyqtSlot()
    def run(self):
        '''
        Initialise the runner function with passed args, kwargs.
        '''
        time_begin = time.time()
        if self.verbose:
            print('Starting process...')

        # Retrieve args/kwargs here; and fire processing using them
        try:
            queue = Queue()
            if self.useMultiProcessing is False:
                # Only use threading, without using multiprocessing.Process
                result = self.fn(*self.args, queue, self.sender, **self.kwargs)
                # result = queue.get()
            else:
                # Process using multiprocessing.Pool
                # This interface is consistent with threading, but progress cannot be communicated.
                # Hence below approach using multiprocessing.Process and Queue is used.
                # p = Pool(processes=1)
                # results = p.map(self.fn, [*self.args])
                # result = results[0]
                # p.close()

                # The function input has 2 additional arguements (Queue and callback function).
                # This interface is not straightforward and needs to be fixed later.

                p = Process(target=self.fn, args=[*self.args, queue, self.sender], kwargs=self.kwargs, daemon=True)
                p.start()

                while queue.empty():
                    try:
                        # progress = self.receiver.poll(timeout=None)
                        progress = None
                        while self.receiver.poll():
                            progress = self.receiver.recv()
                    except EOFError:
                        break
                    else:
                        self.signals.progress.emit(progress)
                result = queue.get()
                p.join()

        except:
            traceback.print_exc()
            exctype, value = sys.exc_info()[:2]
            self.signals.error.emit((exctype, value, traceback.format_exc()))
        else:
            self.signals.result.emit(result)  # Return the result of the processing
            if self.verbose:
                print('Process finished!! Time elapsed = ' + ('%.3f' %(time.time()-time_begin)) + ' sec')
        finally:
            self.signals.finished.emit()  # Done

# Alternative implementation. Unfinished.
# class SimpleThread(QtCore.QThread):
#     finished = QtCore.pyqtSignal(object)

#     def __init__(self, queue, callback, parent=None):
#         QtCore.QThread.__init__(self, parent)
#         self.queue = queue
#         self.finished.connect(callback)

#     def run(self):
#         while True:
#             arg = self.queue.get()
#             if arg is None: # None means exit
#                 print("Shutting down")
#                 return
#             self.fun(arg)

#     def fun(self, sim):
#         print('Running ' + str(sim.sim_time))
#         log = sim.run()
#         print('Finished ' + str(sim.sim_time))
#         self.finished.emit(log)


import json

with open('../features.txt') as f:
    FEATURES = json.loads(f.read())

sites =  ['frontiersin.org','chicagotribune.com','link.springer.com',
          'aljazeera.com','instructables.com','npr.org',
          'dailymail.co.uk','csmonitor.com','baltimoresun.com','city-data.com']

for site in sites:

    feature_dict = {}
    for i in range(FEATURES['c4']['url']):
        feature_dict[i] = 0

    with open('files/'+site+'_data.json','r') as f:
        for line in f:
            data = json.loads(line)
            feature_dict[data['url_id']] += 1

    top_features = sorted(feature_dict, key=feature_dict.get, reverse=True)[:10]

    count = 0
    with open('files/'+site+'_data_filtered.json','w') as f_filtered:
        with open('files/'+site+'_data.json','r') as f:
            for line in f:
                data = json.loads(line)
                if data['url_id'] in top_features:
                    if count != 0:
                        f_filtered.write('\n'+line)
                    else:
                        f_filtered.write(line)
                    count += 1

    print(site)
    print(top_features)
    print(count)
    print()