# -*- coding: utf-8 -*-
"""
Created on Wed Jun 26 02:39:05 2019

@author: NRAD
"""

import cv2
import os

from matplotlib import pyplot as plt

path=r'C:\Users\NRAD.NRAD-28\Desktop\BCDR-F03_dataset-20190626T062636Z-001'

files_list=[]

for root, dirs, files in os.walk(path):
    for file in files:
        if file.endswith(".tif"):
             files_list.append(os.path.join(root, file))

             #print(os.path.join(root, file))

for i in range(len(files_list)):

    img=cv2.imread(files_list[i])

    shape=img.shape[0]

    img=img[200:shape,:,:]

    cv2.imwrite(files_list[i],img)

import pandas as pd

df=pd.read_csv(r"C:\Users\NRAD.NRAD-28\Desktop\BCDR-F03_dataset-20190626T062636Z-001\BCDR-F03_dataset\BCDR-F03\bcdr_f03_features.csv")

truth_values=df['classification']

y_truth_list=[]

for i in range(668):

    index1=files_list[i].rindex('_')

    index2=files_list[i].rindex('_',0,index1)

    index3=files_list[i].rindex('_',0,index2)

    view=files_list[i][index1+1:-4]

    study_id=files_list[i][index2+1:index2+5]

    patient_id=files_list[i][index3+1:index3+4]

    #print(view)

    if(df['patient_id'][i] == patient_id and df['study_id'][i]==study_id):

        y_truth_list.append(df['classification'][i])


import logging
import sys

from lmi.scripts.common import configuration
from lmi.scripts.common import errors

TERM_COLOR_NAMES = (
        'black',
        'red',
        'green',
        'yellow',
        'blue',
        'magenta',
        'cyan',
        'white',
        )

# Following are terminal color codes of normal mode.
CN_BLACK   = 0
CN_RED     = 1
CN_GREEN   = 2
CN_YELLOW  = 3
CN_BLUE    = 4
CN_MAGENTA = 5
CN_CYAN    = 6
CN_WHITE   = 7
# Following are terminal color codes of bright mode.
CB_BLACK   = 8
CB_RED     = 9
CB_GREEN   = 10
CB_YELLOW  = 11
CB_BLUE    = 12
CB_MAGENTA = 13
CB_CYAN    = 14
CB_WHITE   = 15

WARNING_COLOR  = CB_YELLOW
ERROR_COLOR    = CB_RED
CRITICAL_COLOR = CB_MAGENTA

#: Dictionary assigning color code to logging level.
LOG_LEVEL_2_COLOR = {
        logging.WARNING : WARNING_COLOR,
        logging.ERROR   : ERROR_COLOR,
        logging.CRITICAL: CRITICAL_COLOR
}

class LogRecord(logging.LogRecord):
    """
    Overrides :py:class:`logging.LogRecord`. It adds new attributes:

        * `levelname_` - Name of level in lowercase.
        * `cseq`       - Escape sequence for terminal used to set color
                         assigned to particular log level.
        * `creset`     - Escape sequence for terminal used to reset foreground
                         color.

    These can be used in format strings initializing logging formatters.

    Accepts the same arguments as base class.
    """

    def __init__(self, name, level, *args, **kwargs):
        use_colors = kwargs.pop('use_colors', True)
        logging.LogRecord.__init__(self, name, level, *args, **kwargs)
        self.levelname_ = self.levelname.lower()
        if use_colors and level >= logging.WARNING:
            if level >= logging.CRITICAL:
                color = LOG_LEVEL_2_COLOR[logging.CRITICAL]
            elif level >= logging.ERROR:
                color = LOG_LEVEL_2_COLOR[logging.ERROR]
            else:
                color = LOG_LEVEL_2_COLOR[logging.WARNING]
            self.cseq = get_color_sequence(color)
            self.creset = "\x1b[39m"
        else:
            self.cseq = self.creset = ''

class ScriptsLogger(logging.getLoggerClass()):

    #: Boolean saying whether the color sequences for log messages shall be
    #: generated.
    USE_COLORS = True

    def makeRecord(self, name, level, fn, lno, msg, args, exc_info, func=None,
            extra=None):
        """
        Overriden method that just changes the *LogRecord* class to our
        predefined and ensures that exception's traceback is printed once at
        most.
        """
        if exc_info:
            err = exc_info[1]
            if getattr(err, '_traceback_logged', False):
                # do not print traceback second time
                exc_info = None
            elif self.isEnabledFor(level):
                try:
                    err._traceback_logged = True
                except AttributeError:
                    pass
        rv = LogRecord(name, level, fn, lno, msg, args, exc_info, func,
                use_colors=self.USE_COLORS)
        if extra is not None:
            for key in extra:
                if (key in ["message", "asctime"]) or (key in rv.__dict__):
                    raise KeyError("Attempt to overwrite %r in LogRecord" % key)
                rv.__dict__[key] = extra[key]
        return rv

    def exception(self, msg, *args, **kwargs):
        lmi_config = configuration.Configuration.get_instance()
        exc_info = sys.exc_info()
        err = exc_info[1]
        if (   lmi_config.trace
           and (  not isinstance(err, errors.LmiError)
               or (   not getattr(err, '_traceback_logged', False)
                  and lmi_config.verbosity >= lmi_config.OUTPUT_DEBUG))):
            kwargs['exc_info'] = exc_info
        else:
            kwargs.pop('exc_info', None)
        self.error(msg, *args, **kwargs)

class LevelDispatchingFormatter(object):
    """
    Formatter class for logging module. It allows to predefine different format
    string used for some level ranges.

    :param dict formatters: Mapping of module names to *format*.
        It is a dictionary of following format: ::

            { max_level1 : format1
            , max_level2 : format2
            , ... }

        *format* in parameters description can be either ``string`` or another
        formatter object.

        For example if you want to have format3 used for *ERROR* and *CRITICAL*
        levels, *format2* for *INFO* and *format1* for anything else, your
        dictionary will look like this: ::

            { logging.INFO - 1 : format1
            , logging.INFO     : format2 }

        And the *default* value should have *format3* assigned.

    :param default: Default *format* to use. This format is used for all levels
        higher than the maximum of *formatters*' keys.
    """
    def __init__(self, formatters, default=configuration.DEFAULT_FORMAT_STRING,
            datefmt=None):
        for k, formatter in formatters.items():
            if isinstance(formatter, basestring):
                formatters[k] = logging.Formatter(formatter, datefmt=datefmt)
        #: This a a tuple of pairs sorted by the first item in descending
        #: order (highest priority ordered first).
        self._formatters = tuple(sorted(formatters.items(),
            key=lambda t: -t[0]))
        if isinstance(default, basestring):
            default = logging.Formatter(default, datefmt=datefmt)
        self._default_formatter = default

    def format(self, record):
        """
        Interface for logging module.
        """
        record.levelname_ = record.levelname.lower()
        formatter = self._default_formatter
        for level, fmt in self._formatters:
            if level < record.levelno:
                break
            formatter = fmt
        try:
            return formatter.format(record)
        except KeyError:
            # in some modules or libraries it may happen that logger is
            # initialized before our logger class is set as default
            record.cseq = record.creset = ''
            return formatter.format(record)

def get_logger(module_name):
    """
    Convenience function for getting callable returning logger for particular
    module name. It's supposed to be used at module's level to assign its
    result to global variable like this: ::

        from lmi.scripts import common

        LOG = common.get_logger(__name__)

    This can be used in module's functions and classes like this: ::

        def module_function(param):
            LOG().debug("This is debug statement logging param: %s", param)

    Thanks to ``LOG`` being a callable, it always returns valid logger object
    with current configuration, which may change overtime.

    :param string module_name: Absolute dotted module path.
    :rtype: :py:class:`logging.Logger`
    """
    def _logger():
        """ Callable used to obtain current logger object. """
        return logging.getLogger(module_name)
    return _logger

def get_color_sequence(color_code):
    """
    Computer color sequence for particular color code.

    :returns: Escape sequence for terminal used to set foreground color.
    :rtype: str
    """
    if color_code <= 7:
        return "\x1b[%dm" % (30 + color_code)
    return "\x1b[38;5;%dm" % color_code

def setup_logger(use_colors=True):
    """
    This needs to be called before any logging takes place.
    """
    ScriptsLogger.USE_COLORS = use_colors
    logging.setLoggerClass(ScriptsLogger)


# -*- coding: utf-8 -*-
"""
Spyder Editor

This is a temporary script file.
"""
import QuantLib as ql
date1 = ql.Date(1, 1, 2015)
date2 = ql.Date(1, 1, 2016)
tenor = ql.Period(ql.Monthly)
calendar = ql.UnitedStates()
schedule = ql.Schedule(date1, date2, tenor, calendar, ql.Following, ql.Following, ql.DateGeneration.Forward, False)
list(schedule)
#%%
print(date1)
print(list(schedule))


import cv2
import os
import numpy as np
import json
from cam import camera_streams
cap=camera_streams(mode="Industrial")

def hconcat_resize(img_list,
                   interpolation
                   = cv2.INTER_CUBIC):
      # take minimum hights
    h_min = min(img.shape[0]
                for img in img_list)

    # image resizing
    im_list_resize = [cv2.resize(img,
                       (int(img.shape[1] * h_min / img.shape[0]),
                        h_min), interpolation
                                 = interpolation)
                      for img in img_list]

    # return final image
    return cv2.hconcat(im_list_resize)

#function is for concatinate the image vertically

def vconcat_resize(img_list, interpolation
                   = cv2.INTER_CUBIC):
      # take minimum width
    w_min = min(img.shape[1]
                for img in img_list)

    # resizing images
    im_list_resize = [cv2.resize(img,
                      (w_min, int(img.shape[0] * w_min / img.shape[1])),
                                 interpolation = interpolation)
                      for img in img_list]
    # return final image
    return cv2.vconcat(im_list_resize)

def image_loc(foldername):
    # extractte file
    fileName = []
    for root, dirs, files in os.walk(os.path.abspath(foldername)):
        for namef in files:
            #                 print(os.path.abspath(os.path.join(root, namef)))
            file_name = os.path.abspath(os.path.join(root, namef))
            fileName.append(file_name)
    return fileName

good=image_loc("muvro/sona_data")
# good=image_loc("sona_data/wbolt_wpin")

# pin=[[171, 146, 44, 45], [475, 160, 35, 38], [468, 326, 38, 38], [174, 319, 35, 36]]
pin=[[257, 194, 6, 5], [378, 168, 5, 3], [394, 235, 5, 5], [273, 263, 5, 6]]
bolt=[[328, 109, 26, 27], [359, 115, 25, 31], [386, 129, 25, 34], [408, 155, 22, 29], [419, 182, 23, 30], [423, 215, 21, 26], [412, 244, 23, 29], [393, 270, 28, 26], [367, 292, 33, 23], [341, 299, 30, 29], [309, 300, 29, 30], [280, 292, 30, 31], [255, 279, 27, 29], [237, 257, 24, 30], [226, 229, 27, 31], [225, 196, 22, 34], [232, 168, 26, 29], [247, 144, 21, 23], [267, 121, 31, 26], [293, 113, 36, 22]]

# param={"min1": 153, "max1": 219, "min2": 164, "max2": 223, "min3": 184, "max3": 255, "min4": 183, "max4": 259, "min5": 192, "max5": 252, "min6": 138, "max6": 245, "min7": 126, "max7": 237, "min8": 97, "max8": 230, "min9": 85, "max9": 252, "min10": 75, "max10": 210, "min11": 62, "max11": 187, "min12": 51, "max12": 209, "min13": 102, "max13": 253, "min14": 77, "max14": 245, "min15": 88, "max15": 238, "min16": 164, "max16": 256, "min17": 116, "max17": 240, "min18": 71, "max18": 217, "min19": 94, "max19": 256, "min20": 170, "max20": 286}
# param={'min1': 192, 'max1': 196, 'min2': 193, 'max2': 199, 'min3': 240, 'max3': 248, 'min4': 236, 'max4': 243, 'min5': 238, 'max5': 245, 'min6': 232, 'max6': 242, 'min7': 222, 'max7': 232, 'min8': 222, 'max8': 230, 'min9': 241, 'max9': 248, 'min10': 199, 'max10': 206, 'min11': 177, 'max11': 184, 'min12': 197, 'max12': 205, 'min13': 232, 'max13': 240, 'min14': 221, 'max14': 229, 'min15': 230, 'max15': 236, 'min16': 232, 'max16': 240, 'min17': 231, 'max17': 237, 'min18': 173, 'max18': 177, 'min19': 211, 'max19': 216, 'min20': 231, 'max20': 241}

param={}
pins={}
print(len(bolt))
pause=0
# for i in good:
while(1):

    img1=cap.get_frame((680,500))

    img=cv2.cvtColor(img1,cv2.COLOR_BGR2GRAY)

    img2=img.copy()

    rois=[]

    thresh=cv2.threshold(img2,80,255,cv2.THRESH_BINARY)[1]

    cv2.imshow("thresh",thresh)

    k=cv2.waitKey(1)
    if k==ord('s'):
        pause=1
        print("stop")
    if k==ord('p'):
        pause=0
        print('play')
    if pause==0:
        for r,j in zip(bolt,range(1,21)):

            img1=cv2.rectangle(img1,(r[0],r[1]),(r[0]+r[2],r[1]+r[3]),255,1)
            roi = thresh[int(r[1]):int(r[1]+r[3]),
                        int(r[0]):int(r[0]+r[2])]

            bl=cv2.medianBlur(roi, 9)

            roi=cv2.Canny(bl,100,180)

            rh,rw=roi.shape
            n_black=np.count_nonzero(roi == 255)

            if len(param)<40:
                param[f"min{j}"]=n_black
                param[f"max{j}"]=n_black
            else:
                if param[f"min{j}"]>n_black:param[f"min{j}"]=n_black
                if param[f"max{j}"]<n_black:param[f"max{j}"]=n_black
            # print(n_white,n_black,rh*rw)
            rois.append(roi)

            # cv2.imshow(str(j),roi)
        # print("-----------------------------------------------------------------------------------")
        roiss=vconcat_resize([hconcat_resize(rois[:5]),hconcat_resize(rois[5:10]),hconcat_resize(rois[10:15]),hconcat_resize(rois[15:20])])
        roiss=cv2.resize(roiss,(400,400))
        print(param)
        cv2.imshow("roiss",roiss)

        cv2.imshow("image",img1)
    if k ==ord("q"):break
# while True:
#     frame=cap.get_frame((680,500))
#     cv2.imshow("img",img)
#     if cv2.waitKey(200)==ord('q'):break
cv2.destroyAllWindows()
print(param)

# param=json.dumps(param)
# with open("muvro/boltsparam.json", "w") as dic:
#     dic.write(param)

# bolt=json.dumps(bolt)
# with open("muvro/bolts.json", "w") as dic:
#     dic.write(bolt)

# pin=json.dumps(pin)
# with open("muvro/pin.json", "w") as dic:
#     dic.write(pin)

# pins=json.dumps(pins)
# with open("muvro/pinsparam.json", "w") as dic:
    # dic.write(pins)

# img=cap.get_frame((680,500))

# cv2.circle(img,(333,221),90,(255,255,255),-1)
# a = cv2.selectROIs("select the area", img)

# print(a.tolist())

# for r in a:
#     img=cv2.rectangle(img,(r[0],r[1]),(r[0]+r[2],r[1]+r[3]),255,1)
# cv2.imshow("img",img)
# cv2.waitKey(0)
# cv2.destroyAllWindows()


import logging

from sqlalchemy import Column, ForeignKeyConstraint, Integer, Table, Text
from sqlalchemy import PassiveDefault
from sqlalchemy.exceptions import InvalidRequestError
from sqlalchemy.ext.associationproxy import association_proxy
from sqlalchemy.orm import backref, eagerload, relation
from sqlalchemy.orm.collections import attribute_mapped_collection,\
        mapped_collection
from sqlalchemy.sql import and_
from turbogears.database import get_engine, metadata, mapper, session

from fedora.tg.json import SABase
from pkgdb.model.acls import GroupPackageListing, GroupPackageListingAcl,\
        PersonPackageListing, PersonPackageListingAcl

error_log = logging.getLogger('pkgdb.model.packages')

get_engine()

DEFAULT_GROUPS = {'provenpackager': {'commit': True, 'checkout': True}}

#
# Tables
#

# Package and PackageListing are straightforward translations.  Look at these
# if you're looking for a straightforward example.

# :C0103: Tables and mappers are constants but SQLAlchemy/TurboGears convention
# is not to name them with all uppercase
#pylint:disable-msg=C0103
PackageTable = Table('package', metadata, autoload=True)
PackageListingTable = Table('packagelisting', metadata, autoload=True)

#pylint:enable-msg=C0103
#
# Mapped Classes
#

class Package(SABase):
    '''Software we are packaging.

    This is equal to the software in one of our revision control directories.
    It is unversioned and not associated with a particular collection.

    Table -- Package
    '''
    def __init__(self, name, summary, statuscode, description=None,
            reviewurl=None, shouldopen=None, upstreamurl=None):
        #pylint:disable-msg=R0913
        super(Package, self).__init__()
        self.name = name
        self.summary = summary
        self.statuscode = statuscode
        self.description = description
        self.reviewurl = reviewurl
        self.shouldopen = shouldopen
        self.upstreamurl = upstreamurl

    def __repr__(self):
        return 'Package(%r, %r, %r, description=%r, ' \
               'upstreamurl=%r, reviewurl=%r, shouldopen=%r)' % (
                self.name, self.summary, self.statuscode, self.description,
                self.upstreamurl, self.reviewurl, self.shouldopen)

    def api_repr(self, version):
        """ Used by fedmsg to serialize Packages in messages. """
        if version == 1:
            return dict(
                name=self.name,
                summary=self.summary,
                description=self.description,
                reviewurl=self.reviewurl,
                upstreamurl=self.upstreamurl,
            )
        else:
            raise NotImplementedError("Unsupported version %r" % version)

    def create_listing(self, collection, owner, statusname,
            qacontact=None, author_name=None):
        '''Create a new PackageListing branch on this Package.

        :arg collection: Collection that the new PackageListing lives on
        :arg owner: The owner of the PackageListing
        :arg statusname: Status to set the PackageListing to
        :kwarg qacontact: QAContact for this PackageListing in bugzilla.
        :kwarg author_name: Author of the change.  Note: will remove when
            logging is made generic
        :returns: The new PackageListing object.

        This creates a new PackageListing for this Package.  The PackageListing
        has default values set for group acls.
        '''
        from pkgdb.lib.utils import STATUS
        from pkgdb.model.logs import PackageListingLog
        pkg_listing = PackageListing(owner, STATUS[statusname],
                collectionid=collection.id,
                qacontact=qacontact)
        pkg_listing.packageid = self.id
        for group in DEFAULT_GROUPS:
            new_group = GroupPackageListing(group)
            #pylint:disable-msg=E1101
            pkg_listing.groups2[group] = new_group
            #pylint:enable-msg=E1101
            for acl, status in DEFAULT_GROUPS[group].iteritems():
                if status:
                    acl_statuscode = STATUS['Approved']
                else:
                    acl_statuscode = STATUS['Denied']
                group_acl = GroupPackageListingAcl(acl, acl_statuscode)
                # :W0201: grouppackagelisting is added to the model by
                #   SQLAlchemy so it doesn't appear in __init__
                #pylint:disable-msg=W0201
                group_acl.grouppackagelisting = new_group
                #pylint:enable-msg=W0201

        # Create a log message
        log = PackageListingLog(author_name, STATUS['Added'],
                '%(user)s added a %(branch)s to %(pkg)s' %
                {'user': author_name, 'branch': collection,
                    'pkg': self.name})
        log.listing = pkg_listing

        return pkg_listing

class PackageListing(SABase):
    '''This associates a package with a particular collection.

    Table -- PackageListing
    '''
    def __init__(self, owner, statuscode, packageid=None, collectionid=None,
            qacontact=None, specfile=None):
        #pylint:disable-msg=R0913
        super(PackageListing, self).__init__()
        self.packageid = packageid
        self.collectionid = collectionid
        self.owner = owner
        self.qacontact = qacontact
        self.statuscode = statuscode
        self.specfile = specfile

    packagename = association_proxy('package', 'name')

    def __repr__(self):
        return 'PackageListing(%r, %r, packageid=%r, collectionid=%r,' \
               ' qacontact=%r, specfile=%r)' % (self.owner, self.statuscode,
                        self.packageid, self.collectionid, self.qacontact,
                        self.specfile)

    def api_repr(self, version):
        """ Used by fedmsg to serialize PackageListing in messages. """
        if version == 1:
            return dict(
                package=self.package.api_repr(version),
                collection=self.collection.api_repr(version),
                owner=self.owner,
                qacontact=self.qacontact,
                specfile=self.specfile,
            )
        else:
            raise NotImplementedError("Unsupported version %r" % version)

    def clone(self, branch, author_name):
        '''Clone the permissions on this PackageListing to another `Branch`.

        :arg branch: `branchname` to make a new clone for
        :arg author_name: Author of the change.  Note, will remove when logs
            are made generic
        :raises sqlalchemy.exceptions.InvalidRequestError: when a request
            does something that violates the SQL integrity of the database
            somehow.
        :returns: new branch
        :rtype: PackageListing
        '''
        from pkgdb.lib.utils import STATUS
        from pkgdb.model.collections import Branch
        from pkgdb.model.logs import GroupPackageListingAclLog, \
                PersonPackageListingAclLog
        # Retrieve the PackageListing for the to clone branch
        try:
            #pylint:disable-msg=E1101
            clone_branch = PackageListing.query.join('package'
                    ).join('collection').filter(
                        and_(Package.name==self.package.name,
                            Branch.branchname==branch)).one()
            #pylint:enable-msg=E1101
        except InvalidRequestError:
            ### Create a new package listing for this release ###

            # Retrieve the collection to make the branch for
            #pylint:disable-msg=E1101
            clone_collection = Branch.query.filter_by(branchname=branch).one()
            #pylint:enable-msg=E1101
            # Create the new PackageListing
            clone_branch = self.package.create_listing(clone_collection,
                    self.owner, STATUS[self.statuscode], qacontact=self.qacontact,
                    author_name=author_name)

        log_params = {'user': author_name,
                'pkg': self.package.name, 'branch': branch}
        # Iterate through the acls in the master_branch
        #pylint:disable-msg=E1101
        for group_name, group in self.groups2.iteritems():
        #pylint:enable-msg=E1101
            log_params['group'] = group_name
            if group_name not in clone_branch.groups2:
                # Associate the group with the packagelisting
                #pylint:disable-msg=E1101
                clone_branch.groups2[group_name] = \
                        GroupPackageListing(group_name)
                #pylint:enable-msg=E1101
            clone_group = clone_branch.groups2[group_name]
            for acl_name, acl in group.acls2.iteritems():
                if acl_name not in clone_group.acls2:
                    clone_group.acls2[acl_name] = \
                            GroupPackageListingAcl(acl_name, acl.statuscode)
                else:
                    # Set the acl to have the correct status
                    if acl.statuscode != clone_group.acls2[acl_name].statuscode:
                        clone_group.acls2[acl_name].statuscode = acl.statuscode

                # Create a log message for this acl
                log_params['acl'] = acl.acl
                log_params['status'] = acl.status.locale['C'].statusname
                log_msg = '%(user)s set %(acl)s status for %(group)s to' \
                        ' %(status)s on (%(pkg)s %(branch)s)' % log_params
                log = GroupPackageListingAclLog(author_name,
                        acl.statuscode, log_msg)
                log.acl = clone_group.acls2[acl_name]

        #pylint:disable-msg=E1101
        for person_name, person in self.people2.iteritems():
        #pylint:enable-msg=E1101
            log_params['person'] = person_name
            if person_name not in clone_branch.people2:
                # Associate the person with the packagelisting
                #pylint:disable-msg=E1101
                clone_branch.people2[person_name] = \
                        PersonPackageListing(person_name)
                #pylint:enable-msg=E1101
            clone_person = clone_branch.people2[person_name]
            for acl_name, acl in person.acls2.iteritems():
                if acl_name not in clone_person.acls2:
                    clone_person.acls2[acl_name] = \
                            PersonPackageListingAcl(acl_name, acl.statuscode)
                else:
                    # Set the acl to have the correct status
                    if clone_person.acls2[acl_name].statuscode \
                            != acl.statuscode:
                        clone_person.acls2[acl_name].statuscode = acl.statuscode
                # Create a log message for this acl
                log_params['acl'] = acl.acl
                log_params['status'] = acl.status.locale['C'].statusname
                log_msg = '%(user)s set %(acl)s status for %(person)s to' \
                        ' %(status)s on (%(pkg)s %(branch)s)' % log_params
                log = PersonPackageListingAclLog(author_name,
                        acl.statuscode, log_msg)
                log.acl = clone_person.acls2[acl_name]

        return clone_branch

def collection_alias(pkg_listing):
    '''Return the collection_alias that a package listing belongs to.

    :arg pkg_listing: PackageListing to find the Collection for.
    :returns: Collection Alias.  This is either the branchname or a combination
        of the collection name and version.

    This is used to make Branch keys for the dictionary mapping of pkg listings
    into packages.
    '''
    return pkg_listing.collection.simple_name

#
# Mappers
#

mapper(Package, PackageTable, properties={
    # listings is here for compatibility.  Will be removed in 0.4.x
    'listings': relation(PackageListing),
    'listings2': relation(PackageListing,
        backref=backref('package'),
        collection_class=mapped_collection(collection_alias))
    })

mapper(PackageListing, PackageListingTable, properties={
    'people': relation(PersonPackageListing),
    'people2': relation(PersonPackageListing, backref=backref('packagelisting'),
        collection_class = attribute_mapped_collection('username')),
    'groups': relation(GroupPackageListing),
    'groups2': relation(GroupPackageListing, backref=backref('packagelisting'),
        collection_class = attribute_mapped_collection('groupname')),
    })

{'STATUS': 'pkgdb.lib.utils.STATUS', 'Branch': 'pkgdb.model.collections.Branch', 'GroupPackageListingAclLog': 'pkgdb.model.logs.GroupPackageListingAclLog', 'PersonPackageListingAclLog': 'pkgdb.model.logs.PersonPackageListingAclLog'}.query.join

{'STATUS': 'pkgdb.lib.utils.STATUS', 'PackageListingLog': 'pkgdb.model.logs.PackageListingLog'}.name


'''
1.最远点采样
2.open3d和matplotlib对点云的可视化
3.加载pointconv模型、加载点云实例模型、输入并预测
'''
from tqdm import tqdm
from model.pointconv import PointConvDensityClsSsg as PointConvClsSsg
import time
import numpy as np
import torch.utils.data
from data_utils.ModelNetDataLoader import ModelNetDataLoader
import matplotlib.pyplot as plt
import open3d

def farthest_point_sample(xyz, npoint):
    """
    Input:
        xyz: pointcloud data, [B, N, C]
        npoint: number of samples
    Return:
        centroids: sampled pointcloud index, [B, npoint]
    """
    B, N, C = xyz.shape
    centroids = np.zeros((B, npoint), dtype=np.long)
    distance = np.ones((B, N), dtype=np.float) * 1e5
    farthest = np.zeros(B, dtype=np.long)
    for i in range(npoint):
        centroids[:, i] = farthest
        centroid = xyz[:, farthest, :].reshape((B, 1, 3))
        dist = np.sum((xyz - centroid) ** 2, -1)
        mask = dist < distance
        distance[mask] = dist[mask]   # 更新所有点到已采样点集合的最短距离
        farthest = np.argmax(distance, -1)  # 选取其中距离最远点作为下一个采样点
    return centroids

def plot_3d(points):
    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    ax.scatter(points[:, 0],  # x
               points[:, 2],  # y
               points[:, 1],  # z
               c=points[:, 2],  # height data for color
               cmap='Spectral',
               marker="o")
    ax.axis('auto')
    ax.set_zlabel('Z', fontdict={'size': 15, 'color': 'red'})
    ax.set_ylabel('Y', fontdict={'size': 15, 'color': 'red'})
    ax.set_xlabel('X', fontdict={'size': 15, 'color': 'red'})
    plt.show()

def open_3d(source_data, window_name='由红到蓝，模型相似度降低'):
    point_cloud = open3d.geometry.PointCloud()
    point_cloud.points = open3d.utility.Vector3dVector(source_data)
    open3d.visualization.draw_geometries([point_cloud], width=960, height=540, left=480, top=270, window_name=window_name)

def model_load(model="./pointconv_modelnet40-0.916126-0049.pth"):
    '''MODEL LOADING'''
    num_class = 40
    classifier = PointConvClsSsg(num_class)
    checkpoint = torch.load(model, map_location=torch.device("cpu"))
    classifier.load_state_dict(checkpoint['model_state_dict'])
    return classifier

# 读取点云txt文件并最远点采样1024个点
def my_data_load(fn="./data/sample/airplane_0002.txt", num=2048):
    with open(fn, 'r') as f:
        raw = f.readlines()
        for i in range(len(raw)):
            raw[i] = raw[i][:-1].split(",")
            for j in range(6):
                raw[i][j] = float(raw[i][j])
        data = np.array(raw).astype(np.float32)
        data = data.reshape((1, data.shape[0], data.shape[1]))
        sampled = farthest_point_sample(data[:, :, :3], num)
        return data[0, sampled, :].reshape(num, 6)

def my_tqdm_eval(model):
    '''MODEL EVA'''
    DATA_PATH = './data/modelnet40_normal_resampled/'
    TEST_DATASET = ModelNetDataLoader(root=DATA_PATH, npoint=1024, split='test', normal_channel=True)
    testDataLoader = torch.utils.data.DataLoader(TEST_DATASET, batch_size=32, shuffle=False,
                                                 num_workers=0)
    classifier = model_load(model)
    classifier = classifier.eval()
    mean_correct = []
    i = 0
    for batch_id, data in tqdm(enumerate(testDataLoader, 0), total=len(testDataLoader), smoothing=0.9):
        i = i + 1
        print("\n---iter {}".format(i))
        start_time = time.time()
        pointcloud, target = data
        target = target[:, 0]  # shape from (32,1) to (32,)

        points = pointcloud.permute(0, 2, 1)  # (32, 6, 1024)
        points, target = points, target
        with torch.no_grad():
            pred = classifier(points[:, :3, :], points[:, 3:, :])

        demo = np.transpose(points[31, :3, :].numpy(), (1, 0))  # (3, 1024) -> (1024, 3)
        plot_3d(demo)
        pred_choice = pred.data.max(1)[1]
        correct = pred_choice.eq(target.long().data).cpu().sum()

        print("\n")
        for i in range(target.shape[0]):
            print("GT: " + class_names[int(target[i])] + "  ||  Predict: " + class_names[int(pred_choice[i])])

        mean_correct.append(correct.item() / float(points.size()[0]))
        print("cost: {} sec".format(time.time() - start_time))

    accuracy = np.mean(mean_correct)
    print('Total Accuracy: %f' % accuracy)

trained_model = "C:/Users/Sean/Desktop/pointconv_pytorch-master/pointconv_modelnet40-0.916126-0049.pth"
class_names = ['airplane', 'bathtub', 'bed', 'bench', 'bookshelf', 'bottle', 'bowl', 'car', 'chair', 'cone',
                   'cup', 'curtain', 'desk', 'door', 'dresser', 'flower_pot', 'glass_box', 'guitar', 'keyboard', 'lamp',
                   'laptop',
                   'mantel', 'monitor', 'night_stand', 'person', 'piano', 'plant', 'radio', 'range_hood', 'sink',
                   'sofa', 'stairs',
                   'stool', 'table', 'tent', 'toilet', 'tv_stand', 'vase', 'wardrobe', 'xbox']

def load_models_show(list, num=400):
    data = []
    offset = 0
    for fn in list:
        model = my_data_load(fn, num)[:, :3]
        model[:, 2] += offset  # Z 轴
        offset += 2.2
        data.append(model)

    data = np.array(data).astype(np.float32).reshape((len(data) * num, 3))
    data[:, 2] = -data[:, 2]
    open_3d(data)

if __name__ == '__main__':

    # data = my_data_load(num=512)
    # plot_3d(data)
    # my_tqdm_eval(trained_model)

    fnlist = ['airplane_0699', 'airplane_0650', 'airplane_0682', 'airplane_0705', 'airplane_0695']  # 无翼尖为top 1
    # fnlist = ['airplane_0699'] # 无旋翼
    prefix = './data/modelnet40_normal_resampled/airplane/'
    for i in range(len(fnlist)):
        fnlist[i] = prefix + fnlist[i] + '.txt'

    load_models_show(fnlist)

    """
    with open("data/modelnet40_normal_resampled/modelnet40_test.txt","r") as f:
        fns = f.readlines()
        index32 = 0
        batch = 1
        fn_dict = {}
        class_batch_dict = {}
        for key in class_names:
            class_batch_dict[key] = set()
        for fn in fns:
            fn_dict[fn[:-1]] = "{} {}".format(batch, index32)
            class_batch_dict[fn[:fn.rfind("_")]].add(batch)
            index32 += 1
            if index32 == 32:
                batch += 1
                index32 = 0

    for k in fn_dict.keys():
        print(k + " " + fn_dict[k])

    for k in class_batch_dict.keys():
        print(k + " " + str(class_batch_dict[k]))
    """


import networkx as nx
from random import uniform, sample, choice
import itertools

# Erdős–Rényi models
def ER_model_Gnp(number_of_nodes=100, edge_probability=.01):
    g = nx.Graph()
    g.add_nodes_from(range(number_of_nodes))
    for el in itertools.combinations(range(number_of_nodes), 2):
        if uniform(0, 1) < edge_probability:
            g.add_edge(*el)
    return g

def ER_model_GnM(number_of_nodes=100, number_of_edges=100): # clearly number_of_edges has to be <= n(n-1)/2
    g = nx.Graph()
    g.add_nodes_from(range(number_of_nodes))
    sampled_edges = sample([pair for pair in itertools.combinations(range(number_of_nodes), 2)], number_of_edges)
    g.add_edges_from(sampled_edges)
    return g

# Watts–Strogatz model
def WS_model(number_of_nodes=100, mean_degree=6, beta=0.01):
    # number_of_nodes >> mean degree >> log(number of nodes) >> 1
    # beta between 0 and 1
    # mean degree has to be an even integer
    g = nx.Graph()
    g.add_nodes_from(range(number_of_nodes))
    for i, j in itertools.combinations(range(number_of_nodes), 2):
        if abs(i-j) % (number_of_nodes - 1 - mean_degree/2) <= mean_degree/2:
            g.add_edge(*(i, j))
    for i in g.nodes:
        for j in range(i+1, i+1+int(mean_degree/2)):
            if uniform(0, 1) < beta:
                g.remove_edge(i, j % number_of_nodes)
                neighbors = list(g.neighbors(i))
                neighbors.append(i)
                g.add_edge(*(i, choice(list(set(g.nodes)-set(neighbors)))))
    return g

# Barabási-Albert model
def BA_model(number_of_nodes=30, alpha=1.):
    g = nx.Graph()
    g.add_nodes_from(range(2))
    g.add_edge(*(0, 1))
    while g.number_of_nodes() < number_of_nodes:
        den = sum([g.degree(node)**alpha for node in g.nodes])
        probabilities = [g.degree(node)**alpha/den for node in g.nodes]
        new_node = g.number_of_nodes() # label of new node
        g.add_node(new_node)
        for i in range(new_node):
            if uniform(0, 1) < probabilities[i]:
                g.add_edge(*(new_node, i))
    return g

def custom_barabasi(n=50, alpha=1., p_opp=.01, p_m=.1, t=0.):
    '''
    This model is based on the Barabasi Albert one, but:
    p_opp = probability to swap the preferential attachment of a node
    p_m = influence of preferential dis-attachment: p_m=1 means that a node could die with probability degree / sum(degrees)
    t = influence of the age of the node. Higher t means that most recent nodes has priority in preferential attachment
    '''
    g = nx.Graph()
    g.add_nodes_from(range(2))
    g.add_edge(*(0, 1))
    while g.number_of_nodes() < n:
        den = sum([(node**t)*(g.degree(node)**alpha) for node in g.nodes])
        probabilities = [(node**t)*(g.degree(node)**alpha)/den for node in g.nodes]
        if uniform(0, 1) < p_opp:
            probabilities = [1. - el for el in probabilities]
        new_node = g.number_of_nodes() # label of new node
        g.add_node(new_node)
        for i in range(new_node):
            if uniform(0, 1) < probabilities[i]:
                g.add_edge(*(new_node, i))
    node_to_remove = []
    for node in g.nodes:
        if uniform(0, 1) < p_m*g.degree(node)/sum([g.degree(n) for n in g.nodes]):
            node_to_remove.append(node)
    for node in node_to_remove:
        g.remove_node(node)
    return g


import scrapy
import re

from Petharbor.items import AustinAnimalCenterItem

class AnimalCenterSpider(scrapy.Spider):
    name="austinanimalcenter"
    allowed_domains = ["petharbor.com"]
    #start_urls = ["http://www.petharbor.com/results.asp?searchtype=ADOPT&stylesheet=include/default.css&frontdoor=1&grid=1&friends=1&samaritans=1&nosuccess=0&rows=5000&imght=120&imgres=thumb&tWidth=200&view=sysadm.v_animal&fontface=arial&fontsize=10&miles=20&lat=30.300474&lon=-97.747247&shelterlist='ASTN','90343','72312','73748','73064','63250','78098','72699','78456','78552','79815','80971','88570','86145','75304','78347','83335','85034','71872','76816','82004','89469','73793','87542','69562','87125','84976','86555','73895','85366','82718','79604','78842','77070','69865','86866'&atype=dog&ADDR=undefined&nav=1&start=4&nomax=1&page=1&where=type_DOG"]

    #From austin animal center in table format with kennel number
    start_urls= ["http://www.petharbor.com/results.asp?searchtype=ADOPT&friends=1&samaritans=1&nosuccess=0&rows=1000&imght=120&imgres=thumb&tWidth=200&view=sysadm.v_austin&nobreedreq=1&bgcolor=ffffff&text=29abe2&link=024562&alink=017db3&vlink=017db3&fontface=arial&fontsize=10&col_hdr_bg=29abe2&col_hdr_fg=29abe2&col_fg=29abe2&SBG=ffffff&zip=78704&miles=10&shelterlist=%27ASTN%27&atype=&where=type_DOG,size_s&PAGE=1",
    "http://www.petharbor.com/results.asp?searchtype=ADOPT&friends=1&samaritans=1&nosuccess=0&rows=1000&imght=120&imgres=thumb&tWidth=200&view=sysadm.v_austin&nobreedreq=1&bgcolor=ffffff&text=29abe2&link=024562&alink=017db3&vlink=017db3&fontface=arial&fontsize=10&col_hdr_bg=29abe2&col_hdr_fg=29abe2&col_fg=29abe2&SBG=ffffff&zip=78704&miles=10&shelterlist=%27ASTN%27&atype=&where=type_DOG,size_m&PAGE=1",
    "http://www.petharbor.com/results.asp?searchtype=ADOPT&friends=1&samaritans=1&nosuccess=0&rows=1000&imght=120&imgres=thumb&tWidth=200&view=sysadm.v_austin&nobreedreq=1&bgcolor=ffffff&text=29abe2&link=024562&alink=017db3&vlink=017db3&fontface=arial&fontsize=10&col_hdr_bg=29abe2&col_hdr_fg=29abe2&col_fg=29abe2&SBG=ffffff&zip=78704&miles=10&shelterlist=%27ASTN%27&atype=&where=type_DOG,size_l&PAGE=1"]

    def parse(self,response):
        #with open("petharbor-scrape.txt","wb") as f:
        #    f.write(response.body)
        #items=[]
        sizes = ['size_s','size_m','size_l']
        size_map = { 'size_s' : 'small' , 'size_m' : 'medium' , 'size_l':'large'}
        for sel in response.css('.ResultsTable').css('tr'):
            item = AustinAnimalCenterItem()

            id = sel.css('td:nth-child(2)').xpath('text()').extract()
            if len(id):
                item["id"] = id[0]
            else:
                continue

            kennel = sel.css('td:nth-child(5)').xpath('text()').extract()
            if len(kennel):
                item["kennel"] = kennel[0]

            for size in sizes:
                p = re.compile(size);
                m = p.search(response.url)

                if m is not None:
                    item["size"] = size_map[size]
                    break

            yield(item)


from pyspark import SparkConf, SQLContext
from pyspark.sql import SparkSession
from pyspark.sql.functions import *
import os

def main():
    """
    Simple Aggregrators
    """
    DATA_PATH = "/Users/akshayterode/Desktop/DEV/CS-BigData/00-Data"
    invoiceDf = spark.read.\
        option("header", True)\
            .option("inferSchema", True)\
                .csv(f"{DATA_PATH}/OrderData.csv")

    # Column Object Expression
    invoiceDf.select(
        count("*").alias("RowCount"), sum("Quantity").alias("TotalQuantity"), avg("UnitPrice").alias("AvgPrice"), countDistinct("InvoiceNo").alias("CountDistinct")
    ).show()

    # String Expression
    invoiceDf.selectExpr(
        "count(*) as RowCount",
        "sum(Quantity) as TotalQuantity",
        "avg(UnitPrice) as AvgPrice",
        "count(Distinct(InvoiceNo)) as CountDistint"
    ).show()

    invoiceDf.selectExpr(
        "count(StockCode) as RowCount",
        "sum(Quantity) as TotalQuantity",
        "avg(UnitPrice) as AvgPrice",
        "count(Distinct(InvoiceNo)) as CountDistint"
    ).show()

    # SQL Version

    invoiceDf.createOrReplaceTempView("sales")
    """
    Query to be used when values are needed in nearest two precision points.
    SELECT
    COUNT(*) as RowCount, CAST(SUM(Quantity) AS DECIMAL(10,2)) as QuantitySum,
    CAST(AVG(UnitPrice) AS DECIMAL(10,2)) as AvgUnitPrice, COUNT(DISTINCT(InvoiceNo)) as DistinctInvoiceNo
    FROM sales;
    """
    spark.sql(
        "SELECT COUNT(*) AS RowCount, SUM(Quantity) AS QuantitySum, AVG(UnitPrice) AS AvgUnitPrice, COUNT(DISTINCT(InvoiceNo)) AS DistinctInvoiceNo FROM sales;"
    ).show()

if __name__ == '__main__':
    spark_conf = SparkConf()
    spark_conf.set("spark.app.name", "DataFrameRunTimeErrorExample")
    spark_conf.set("spark.master", "local[2]")
    spark = SparkSession.builder.config(conf=spark_conf).getOrCreate()
    main()
    spark.stop()


from django.shortcuts import render

from django.http import HttpResponse
from archive.models import Article
from django.http import HttpResponseRedirect
from django.contrib import messages
import os
# Create your views here.

def hello_world(request):
    return HttpResponse("hello world")

def article_content(request):
    article = Article.objects.all()[0]
    article_id = article.article_id
    title = article.title
    author = article.author
    abstract = article.abstract
    publish_date = article.publish_date

    return_str = title
    return HttpResponse(return_str)

def get_root(request):
    return render(request,'archive/index.html')

def get_upload_file(request):
    return render(request,'archive/upload.html')

def get_index_page(request):
    all_article = Article.objects.all()
    return render(request,'archive/classification.html',
                  {
                        'article_list':all_article
                    })

def search(request):
    #print("run here")
    all_article = Article.objects.all()
    #print(all_article)
    q = request.GET.get('q')
    if not q:
        error_msg = '请输入关键词再进行搜索'
        return render(request, 'archive/error.html', {'error_msg': error_msg})
    print(q)
    #q = "article: "+q
    post_list = all_article.filter(title__icontains=q)
    print(post_list)
    if post_list:
        return render(request, 'archive/result.html', {'article_list': post_list})
    error_msg = '没有找到相关文章'
    return render(request, 'archive/error.html', {'error_msg': error_msg})

def upload_file(request):
    if request.method == "POST":    # 请求方法为POST时，进行处理
        myFile =request.FILES.get("myfile", None)    # 获取上传的文件，如果没有文件，则默认为None
        print(myFile)
        if not myFile:
            print("upload failed")
            return HttpResponse("no files for upload!")
        destination = open('D:/upload_files/'+myFile.name,'wb+')    # 打开特定的文件进行二进制的写操作
        for chunk in myFile.chunks():      # 分块写入文件
            destination.write(chunk)
        destination.close()
        print("upload success")
        #messages.success(request,"upload success")
        return HttpResponse("上传成功!")


from typing import Optional, List

# Definition for a binary tree node.
class TreeNode:
    def __init__(self, val=0, left=None, right=None):
        self.val = val
        self.left = left
        self.right = right

class Solution:
    def __init__(self):
        self.result = []
        self.record = {}

    def traversal(self, node):
        if node is None:
            return "#"

        left_subtree = self.traversal(node.left)
        right_subtree = self.traversal(node.right)
        # 存储左右子树加自己的序列化结构，用string，中间用分隔符号，不然会出现22 == 2,2其实是两中结构
        subtree = left_subtree + ',' + right_subtree + ',' + str(node.val)
        # 拿到出现频率，出现一次记录一次就好，不需要重复记录，所有只判断freq是不是1
        freq = self.record.get(subtree, 0)
        if freq == 1:
            self.result.append(node)

        self.record[subtree] = freq + 1

        return subtree

    def findDuplicateSubtrees(self, root: Optional[TreeNode]) -> List[Optional[TreeNode]]:
        """
        Time O(n^2) 遍历每个节点，每个节点需要粗略遍历所有子节点记录下来
        Space O(n) 额外的存储子树空间
        后续遍历，每次在中间节点判断左右子树加自己是不是出现过，用全局dictionary来存储出现的子树结构，
        这里存储方式用string，来判断是否有相同的子树结构
        """
        self.traversal(root)

        return self.result

node1 = TreeNode(2)
node2 = TreeNode(1)
node3 = TreeNode(1)
node1.left = node2
node1.right = node3

s = Solution()
print(s.findDuplicateSubtrees(root=node1))


from rest_framework.fields import SerializerMethodField
from rest_framework.relations import HyperlinkedIdentityField
from rest_framework.serializers import ModelSerializer
from account.serializers import UserOtherDetailsSerializer
from answers.models import Answer
from comments.models import Comment

class CommentDetailsSerializer(ModelSerializer):
    writer = UserOtherDetailsSerializer(read_only=True)
    parent_answer = SerializerMethodField()
    replies = HyperlinkedIdentityField(view_name='children_comment')
    replies_count = SerializerMethodField()

    class Meta:
        model = Comment
        fields = [
            'id',
            'writer',
            'body',
            'no_of_upvotes',
            'timestamp',
            'replies',
            'replies_count',
            'parent_answer',
        ]
        read_only_fields = [
            'no_of_upvotes',
        ]

    def get_replies(self, obj):
        if obj.is_parent:
            return CommentChildSerializer(obj.children(), many=True).data
        return None

    def get_replies_count(self, obj):
        if obj.is_parent:
            return obj.children().count()
        return 0

    def get_parent_answer(self, obj):
        if obj.parent_answer is not None:
            return CommentParentAnswerSerializer(obj.parent_answer).data
        return None

class CommentChildSerializer(ModelSerializer):
    writer = UserOtherDetailsSerializer(read_only=True)
    replies = HyperlinkedIdentityField(view_name='children_comment')
    replies_count = SerializerMethodField()

    class Meta:
        model = Comment
        fields = [
            'id',
            'writer',
            'body',
            'no_of_upvotes',
            'parent',
            'replies',
            'replies_count'
        ]

        read_only_fields = [
            'no_of_upvotes'
        ]

    def get_replies(self, obj):
        if obj.is_parent:
            return CommentChildSerializer(obj.children(), many=True).data
        return None

    def get_replies_count(self, obj):
        try:
            return obj.children().count()
        except:
            return 0

class CommentParentAnswerSerializer(ModelSerializer):
    writer = UserOtherDetailsSerializer(read_only=True)

    class Meta:
        model = Answer
        fields = [
            'writer',
            'body'
        ]


import cv2
import pathlib

# tutor: https://www.youtube.com/watch?v=5cg_yggtkso

print("Face detection with OpenCV")
cascade_path = pathlib.Path(cv2.__file__).parent.absolute() / "data/haarcascade_frontalface_default.xml"
print("Classifier file path: " + str(cascade_path))

clf = cv2.CascadeClassifier(str(cascade_path))

camera = cv2.VideoCapture(0)
# camera = cv2.VideoCapture('video.mp4')

while True:
  _, frame = camera.read()
  gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
  faces = clf.detectMultiScale(
    gray,
    scaleFactor = 1.1,
    minNeighbors = 5,
    minSize = (100, 100),
    flags = cv2.CASCADE_SCALE_IMAGE
  )

  for (x, y, w, h) in faces:
    cv2.rectangle(frame, (x, y), (x+w, y+h), (0, 0, 255), 2)

  cv2.imshow('frame', frame)

  if cv2.waitKey(1) & 0xFF == ord('q'):
    break

camera.release()
cv2.destroyAllWindows()


import pytest

from shutil import copyfile

from tools import cmpFiles, getGuiItem

from PyQt5.QtCore import Qt
from PyQt5.QtWidgets import (
    QDialogButtonBox, QDialog, QAction, QFileDialog, QFontDialog
)

from novelwriter import CONFIG, SHARED
from novelwriter.dialogs.quotes import GuiQuoteSelect
from novelwriter.dialogs.preferences import GuiPreferences

KEY_DELAY = 1

@pytest.mark.gui
def testDlgPreferences_Main(qtbot, monkeypatch, nwGUI, tstPaths):
    """Test the preferences dialog."""
    monkeypatch.setattr(GuiPreferences, "exec_", lambda *a: None)
    monkeypatch.setattr(GuiPreferences, "result", lambda *a: QDialog.Accepted)
    monkeypatch.setattr(SHARED._spelling, "listDictionaries", lambda: [("en", "English [en]")])

    with monkeypatch.context() as mp:
        mp.setattr(GuiPreferences, "updateTheme", lambda *a: True)
        mp.setattr(GuiPreferences, "updateSyntax", lambda *a: True)
        mp.setattr(GuiPreferences, "needsRestart", lambda *a: True)
        mp.setattr(GuiPreferences, "refreshTree", lambda *a: True)
        nwGUI.mainMenu.aPreferences.activate(QAction.Trigger)
        qtbot.waitUntil(lambda: getGuiItem("GuiPreferences") is not None, timeout=1000)

    nwPrefs = getGuiItem("GuiPreferences")
    assert isinstance(nwPrefs, GuiPreferences)
    nwPrefs.show()

    assert nwPrefs.updateTheme is False
    assert nwPrefs.updateSyntax is False
    assert nwPrefs.needsRestart is False
    assert nwPrefs.refreshTree is False

    # General Settings
    qtbot.wait(KEY_DELAY)
    tabGeneral = nwPrefs.tabGeneral
    nwPrefs._tabBox.setCurrentWidget(tabGeneral)

    qtbot.wait(KEY_DELAY)
    assert tabGeneral.showFullPath.isChecked()
    qtbot.mouseClick(tabGeneral.showFullPath, Qt.LeftButton)
    assert not tabGeneral.showFullPath.isChecked()

    qtbot.wait(KEY_DELAY)
    assert not tabGeneral.hideVScroll.isChecked()
    qtbot.mouseClick(tabGeneral.hideVScroll, Qt.LeftButton)
    assert tabGeneral.hideVScroll.isChecked()

    qtbot.wait(KEY_DELAY)
    assert not tabGeneral.hideHScroll.isChecked()
    qtbot.mouseClick(tabGeneral.hideHScroll, Qt.LeftButton)
    assert tabGeneral.hideHScroll.isChecked()

    # Check font button
    monkeypatch.setattr(QFontDialog, "getFont", lambda font, obj: (font, True))
    qtbot.mouseClick(tabGeneral.fontButton, Qt.LeftButton)

    qtbot.wait(KEY_DELAY)
    tabGeneral.guiFontSize.setValue(12)

    # Projects Settings
    qtbot.wait(KEY_DELAY)
    tabProjects = nwPrefs.tabProjects
    nwPrefs._tabBox.setCurrentWidget(tabProjects)
    tabProjects.backupPath = "no/where"

    qtbot.wait(KEY_DELAY)
    assert not tabProjects.backupOnClose.isChecked()
    qtbot.mouseClick(tabProjects.backupOnClose, Qt.LeftButton)
    assert tabProjects.backupOnClose.isChecked()

    # qtbot.stop()

    # Check Browse button
    monkeypatch.setattr(QFileDialog, "getExistingDirectory", lambda *a, **k: "")
    assert not tabProjects._backupFolder()
    monkeypatch.setattr(QFileDialog, "getExistingDirectory", lambda *a, **k: "some/dir")
    qtbot.mouseClick(tabProjects.backupGetPath, Qt.LeftButton)

    qtbot.wait(KEY_DELAY)
    tabProjects.autoSaveDoc.setValue(20)
    tabProjects.autoSaveProj.setValue(40)

    # Document Settings
    qtbot.wait(KEY_DELAY)
    tabDocs = nwPrefs.tabDocs
    nwPrefs._tabBox.setCurrentWidget(tabDocs)

    qtbot.wait(KEY_DELAY)
    qtbot.mouseClick(tabDocs.fontButton, Qt.LeftButton)

    qtbot.wait(KEY_DELAY)
    tabDocs.textSize.setValue(13)
    tabDocs.textWidth.setValue(700)
    tabDocs.focusWidth.setValue(900)
    tabDocs.textMargin.setValue(45)
    tabDocs.tabWidth.setValue(45)

    qtbot.wait(KEY_DELAY)
    assert not tabDocs.hideFocusFooter.isChecked()
    qtbot.mouseClick(tabDocs.hideFocusFooter, Qt.LeftButton)
    assert tabDocs.hideFocusFooter.isChecked()

    qtbot.wait(KEY_DELAY)
    assert not tabDocs.doJustify.isChecked()
    qtbot.mouseClick(tabDocs.doJustify, Qt.LeftButton)
    assert tabDocs.doJustify.isChecked()

    # Editor Settings
    qtbot.wait(KEY_DELAY)
    tabEditor = nwPrefs.tabEditor
    nwPrefs._tabBox.setCurrentWidget(tabEditor)

    qtbot.wait(KEY_DELAY)
    assert not tabEditor.showTabsNSpaces.isChecked()
    qtbot.mouseClick(tabEditor.showTabsNSpaces, Qt.LeftButton)
    assert tabEditor.showTabsNSpaces.isChecked()

    qtbot.wait(KEY_DELAY)
    assert not tabEditor.showLineEndings.isChecked()
    qtbot.mouseClick(tabEditor.showLineEndings, Qt.LeftButton)
    assert tabEditor.showLineEndings.isChecked()

    qtbot.wait(KEY_DELAY)
    assert not tabEditor.autoScroll.isChecked()
    qtbot.mouseClick(tabEditor.autoScroll, Qt.LeftButton)
    assert tabEditor.autoScroll.isChecked()

    qtbot.wait(KEY_DELAY)
    tabEditor.scrollPastEnd.setValue(0)

    qtbot.wait(KEY_DELAY)
    tabEditor.bigDocLimit.setValue(500)

    # Syntax Settings
    qtbot.wait(KEY_DELAY)
    tabSyntax = nwPrefs.tabSyntax
    nwPrefs._tabBox.setCurrentWidget(tabSyntax)

    qtbot.wait(KEY_DELAY)
    assert tabSyntax.highlightQuotes.isChecked()
    qtbot.mouseClick(tabSyntax.highlightQuotes, Qt.LeftButton)
    assert not tabSyntax.highlightQuotes.isChecked()

    qtbot.wait(KEY_DELAY)
    assert tabSyntax.highlightEmph.isChecked()
    qtbot.mouseClick(tabSyntax.highlightEmph, Qt.LeftButton)
    assert not tabSyntax.highlightEmph.isChecked()

    # Automation Settings
    qtbot.wait(KEY_DELAY)
    tabAuto = nwPrefs.tabAuto
    nwPrefs._tabBox.setCurrentWidget(tabAuto)

    qtbot.wait(KEY_DELAY)
    assert tabAuto.autoSelect.isChecked()
    qtbot.mouseClick(tabAuto.autoSelect, Qt.LeftButton)
    assert not tabAuto.autoSelect.isChecked()

    qtbot.wait(KEY_DELAY)
    assert tabAuto.doReplace.isChecked()
    qtbot.mouseClick(tabAuto.doReplace, Qt.LeftButton)
    assert not tabAuto.doReplace.isChecked()

    qtbot.wait(KEY_DELAY)
    assert not tabAuto.doReplaceSQuote.isEnabled()
    assert not tabAuto.doReplaceDQuote.isEnabled()
    assert not tabAuto.doReplaceDash.isEnabled()
    assert not tabAuto.doReplaceDots.isEnabled()

    # Quotation Style
    qtbot.wait(KEY_DELAY)
    tabQuote = nwPrefs.tabQuote
    nwPrefs._tabBox.setCurrentWidget(tabQuote)

    monkeypatch.setattr(GuiQuoteSelect, "selectedQuote", "'")
    monkeypatch.setattr(GuiQuoteSelect, "exec_", lambda *a: QDialog.Accepted)
    qtbot.mouseClick(tabQuote.btnDoubleStyleC, Qt.LeftButton)

    # Save and Check Config
    qtbot.mouseClick(nwPrefs.buttonBox.button(QDialogButtonBox.Ok), Qt.LeftButton)
    nwPrefs._doClose()

    assert CONFIG.saveConfig()
    projFile = tstPaths.cnfDir / "novelwriter.conf"
    testFile = tstPaths.outDir / "guiPreferences_novelwriter.conf"
    compFile = tstPaths.refDir / "guiPreferences_novelwriter.conf"
    copyfile(projFile, testFile)
    ignTuple = (
        "timestamp", "font", "lastnotes", "localisation", "geometry",
        "preferences", "projcols", "mainpane", "docpane", "viewpane",
        "outlinepane", "textfont", "textsize", "lastpath", "backuppath"
    )
    assert cmpFiles(testFile, compFile, ignoreStart=ignTuple)

    # Clean up
    nwGUI.closeMain()

    # qtbot.stop()

# END Test testDlgPreferences_Main


#Let's do this the right way...
from globals import *

import life as lfe

#import judgement
#import dialog
#import speech
#import brain
import action

import logging

def get_job(job_id):
	if not job_id in WORLD_INFO['jobs']:
		raise Exception('Job with id \'%s\' does not exist.' % job_id)

	return WORLD_INFO['jobs'][job_id]

def get_job_via_name(name):
	for job_id in WORLD_INFO['jobs']:
		if WORLD_INFO['jobs'][job_id]['name'] == name:
			return job_id

	return False

def get_task(job_id, task_id):
	return get_job(job_id)['tasks'][task_id]

def get_creator(job_id):
	return get_job(job_id)['creator']

def create_job(creator, name, gist='', tier=0, description='Job description needed.', ignore_dupe=False, requirements=[], **kwargs):
	if not ignore_dupe and get_job_via_name(name):
		return False

	_job = {'id': str(WORLD_INFO['jobid'])}
	_job['name'] = name
	_job['gist'] = gist
	_job['tier'] = tier
	_job['description'] = description
	_job['tasks'] = {}
	_job['workers'] = []
	_job['flags'] = {}
	_job['creator'] = creator['id']
	_job['completed'] = False
	_job['flags'] = kwargs
	_job['cancel_on'] = []
	_job['requirements'] = requirements

	WORLD_INFO['jobid'] += 1
	WORLD_INFO['jobs'][_job['id']] = _job

	logging.debug('%s created new job: %s' % (' '.join(creator['name']), name))

	return _job['id']

def delete_job(job_id):
	_job = get_job(job_id)

	for worker in _job['workers']:
		if LIFE[worker]['job'] == job_id:
			LIFE[worker]['job'] = None
			LIFE[worker]['task'] = None

		LIFE[worker]['jobs'].remove(job_id)

	del WORLD_INFO['jobs'][job_id]

def cancel_on(job_id, action):
	_job = get_job(job_id)

	_job['cancel_on'].append(action)

def get_flag(job_id, flag):
	_job = get_job(job_id)

	return _job['flags'][flag]

def reset_job(job_id):
	_job = get_job(job_id)
	_job['workers'] = []
	_job['completed'] = False

	for task in _job['tasks'].values():
		task['completed'] = False
		task['requires'] = _task['_required'][:]

	logging.debug('Job with ID \'%s\' reset.' % job_id)

def add_task(job_id, task_id, name, action, player_action=None, description='Task description needed.', requires=[], max_workers=1, delete_on_finish=False):
	_job = get_job(job_id)

	_task = {'id': str(task_id)}
	_task['name'] = name
	_task['description'] = description
	_task['action'] = action
	_task['player_action'] = player_action
	_task['requires'] = requires
	_task['_required'] = []
	_task['max_workers'] = max_workers
	_task['completed'] = False

	if delete_on_finish:
		_task['max_workers'] = max_workers
	else:
		_task['max_workers'] = -1

	_job['tasks'][str(task_id)] = _task

	return str(task_id)

def remove_worker_from_task(job_id, task_id, worker_id):
	_job = get_job(job_id)

	if LIFE[worker_id]['task'] == task_id:
		LIFE[worker_id]['task'] = None

		logging.debug('Removed worker \'%s\' from task \'%s\' of job \'%s\'.' % (' '.join(LIFE[worker_id]['name']), task_id, worker_id))

def complete_task(life, job_id, task_id):
	_job = get_job(job_id)
	_task = get_task(job_id, task_id)

	if _task['max_workers'] == -1:
		life['completed_tasks'].append(task_id)
		_old_task = life['task']
		life['task'] = get_next_task(life, life['job'])

		if not life['task']:
			life['job'] = None
			life['completed_jobs'].append(job_id)
			leave_job(job_id, life['id'])

		logging.debug('Task \'%s\' of job \'%s\' completed by %s.' % (task_id, job_id, ' '.join(life['name'])))
		return False

	_task['completed'] = True

	for task in _job['tasks']:
		if task_id in _job['tasks'][task]['requires']:
			_job['tasks'][task]['requires'].remove(task_id)
			_job['tasks'][task]['_required'].append(task_id)

	for worker in _job['workers']:
		remove_worker_from_task(job_id, task_id, worker)

	logging.debug('Task \'%s\' of job \'%s\' completed by %s.' % (task_id, job_id, ' '.join(life['name'])))

def get_workers_on_task(job_id, task_id):
	_task = get_task(job_id, task_id)
	_workers = []

	for worker_id in get_job(job_id)['workers']:
		if LIFE[worker_id]['task'] == task_id:
			_workers.append(worker_id)

	return _workers

#def meets_job_requirements(life, job_id):
#	_job = get_job(job_id)
#
#	for req in _job['requirements']:
#		if not action.execute_small_script(life, req):
#			return False
#
#	return True

def get_free_tasks(job_id, local_completed=[]):
	_job = get_job(job_id)
	_free_tasks = []

	for task in _job['tasks']:
		if local_completed:
			_req = _job['tasks'][task]['requires'][:]

			for completed_task in local_completed:
				if completed_task in _req:
					_req.remove(completed_task)

			if _req:
				continue
		else:
			if _job['tasks'][task]['completed'] or _job['tasks'][task]['requires']:
				continue

		if not _job['tasks'][task]['max_workers'] == -1 and len(get_workers_on_task(job_id, task)) >= _job['tasks'][task]['max_workers']:
			continue

		_free_tasks.append(task)

	return _free_tasks

def get_next_task(life, job_id):
	#_job = get_job(job_id)

	for task in get_free_tasks(job_id, local_completed=life['completed_tasks']):
		if not task in life['completed_tasks']:
			return task

	return None

def has_completed_or_rejected_job(life, job_id):
	if job_id in life['completed_jobs'] or job_id in life['rejected_jobs']:
		return True

	return False

def is_candidate(job_id, life_id):
	if job_id in LIFE[life_id]['jobs']:
		return True

	return False

def add_job_candidate(job_id, life_id):
	_job = get_job(job_id)

	if is_candidate(job_id, life_id):
		raise Exception('\'%s\' is already a candidate for job %s.' % (' '.join(LIFE[life_id]['name']), job_id))

	LIFE[life_id]['jobs'].append(job_id)

def join_job(job_id, life_id):
	_job = get_job(job_id)

	if life_id in _job['workers']:
		raise Exception('\'%s\' is already a member of job %s.' % (' '.join(LIFE[life_id]['name']), job_id))

	_job['workers'].append(life_id)
	LIFE[life_id]['job'] = job_id

	logging.debug('%s joined job with ID \'%s\'.' % (' '.join(LIFE[life_id]['name']), job_id))

	return job_id

def leave_job(job_id, life_id, reject=False):
	_job = get_job(job_id)

	if not life_id in _job['workers']:
		raise Exception('\'%s\' is not a member of job %s.' % (' '.join(LIFE[life_id]['name']), job_id))

	_job['workers'].remove(life_id)

	if job_id == LIFE[life_id]['job']:
		LIFE[life_id]['job'] = None

	if reject:
		reject_job(job_id, life_id)

	logging.debug('%s left job with ID \'%s\'.' % (' '.join(LIFE[life_id]['name']), job_id))

def reject_job(job_id, life_id):
	LIFE[life_id]['jobs'].remove(job_id)
	LIFE[life_id]['rejected_jobs'].append(job_id)

def alife_has_job(life):
	return life['job']

def meets_job_requirements(life, job_id):
	for requirement in get_job(job_id)['requirements']:
		if not action.execute_small_script(life, requirement):
			return False

	return True

def _work(life):
	if not life['task']:
		return False

	_job = get_job(life['job'])

	for cancel_on_action in _job['cancel_on']:
		if not action.execute_small_script(life, _task['player_action']):
			delete_job(life['job'])
			return False

	_task = get_task(life['job'], life['task'])

	if 'player' in life:
		return action.execute_small_script(life, _task['player_action'])

	return action.execute_small_script(life, _task['action'])

def work(life):
	if _work(life):
		return complete_task(life, life['job'], life['task'])

	return False


import json
import logging

from cornice import Service
from .models import Users
from pyramid import (
    authentication,
    authorization,
    security,
)
from pyramid.httpexceptions import HTTPFound

import requests

from .constants import cors_policy
from .models import Users

log = logging.getLogger(name="mediapublic.{}".format(__name__))

default_acl = [
    (security.Allow, security.Everyone, 'get'),
    (security.Allow, 'group:admins', 'create'),
    (security.Allow, 'group:staff', 'create'),
    (security.Allow, security.Authenticated, 'create'),
    (security.Allow, 'group:admins', 'delete'),
    (security.Allow, 'group:admins', 'update'),
    (security.Allow, 'group:staff', 'delete'),
    (security.Allow, 'group:staff', 'update'),
]

class Context(object):
    def __init__(self, acl):
        """`acl` is a list of triples (result, principal, permission)"""
        self.__acl__ = acl[:]

def choose_context(request):
    # request.matched_route.name
    if request.registry.settings.get(
            'mediapublic.ignore_authentication'):
        return Context([
            (security.Allow, security.Everyone, 'get'),
            (security.Allow, security.Everyone, 'create'),
            (security.Allow, security.Everyone, 'update'),
            (security.Allow, security.Everyone, 'delete'),
        ])
    acl = default_acl[:]
    if ('usersresource' in request.matched_route.name and
            request.authenticated_userid):
        acl.append(
            (security.Allow, request.authenticated_userid, 'update')
        )
    return Context(acl)

def users_acl(request):
    if request.authenticated_userid:
        pass

def associate_groups(user_id, request):
    # TODO(ryansb): actually associate users/groups using the
    # UserTypes model
    return 'user', 'admin'

authn_policy = authentication.AuthTktAuthenticationPolicy(
    # TODO(ryansb): load this from config
    'secretsecrets',
    callback=associate_groups,
    hashalg='sha512',
)

authz_policy = authorization.ACLAuthorizationPolicy()

# --------- Auth and login
login = Service(name='login', path='/login',
                description="Auth and such",
                cors_policy=cors_policy)

@login.get()
def login_info(request):
    if (request.authenticated_userid is None):
        return {}
    user = Users.get_by_id(request.authenticated_userid)
    if user is None:
        request.response.status = 404
        return {'error': 'Not found'}
    return user.to_dict()

@login.post()
def logged_in(request):
    log.debug("Received auth, token ID %s" % request.params['token'])
    resp = requests.get(request.host_url + '/auth/auth_info', params={
        'format': 'json',
        'token': request.params['token'],
    })
    # Future feature: when using providers other than twitter, check
    # auth_info['provider_name'] to see where the account is from
    auth_info = resp.json()
    twitter_handle = auth_info["profile"]["accounts"][0]['username']
    log.debug("Login for @%s" % twitter_handle)

    log.debug("Auth data: %s" % json.dumps(auth_info, indent=4))
    already_exists, uid = Users.update_social_login(twitter_handle, auth_info)

    log.debug("Succesfully authenticated @%s with twitter, exists:%s" %
              (twitter_handle, already_exists))

    principals = security.remember(request, str(uid))
    headers = request.response.headerlist
    headers.extend(principals)
    log.debug("User authenticated, sending back %s" % headers)

    if already_exists:
        return HTTPFound(
            location=request.registry.settings['mediapublic.login_url'],
            headers=headers)
    else:
        return HTTPFound(
            location=request.registry.settings['mediapublic.signup_url'],
            headers=headers)

@login.delete()
def logout(request):
    principals = security.forget(request)
    request.response.headerlist.extend(principals)
    return {"success": True}


from typing import Optional

from ice.evaluation.evaluate_recipe_result import RecipeResult
from ice.metrics.gold_standards import get_gold_standard
from ice.metrics.gold_standards import list_experiments
from ice.paper import get_full_document_id
from ice.paper import Paper
from ice.recipe import Recipe
from ice.recipes.evaluate_result import EvaluateResult
from ice.trace import recorder
from ice.utils import truncate_by_tokens

QUESTION_SHORT_NAME = "placebo"

DEFAULT_ANSWER_CLASSIFICATION = "Placebo"

def get_paper_text(paper: Paper) -> str:
    return "\n\n".join(str(p) for p in paper.paragraphs)

def create_recipe_result(paper_id: str, experiment: str, answer: str) -> RecipeResult:
    return RecipeResult(
        document_id=paper_id,
        question_short_name=QUESTION_SHORT_NAME,
        experiment=experiment,
        classifications=[
            DEFAULT_ANSWER_CLASSIFICATION,
            DEFAULT_ANSWER_CLASSIFICATION,
        ],
        answer=answer,
        result=answer,
        excerpts=[],
    )

def get_gold_placebo(paper_id: str, experiment: str) -> Optional[str]:
    """
    Return the gold standard placebo description for the given paper and experiment.
    """
    gold_standard = get_gold_standard(
        document_id=paper_id, question_short_name="placebo", experiment=experiment
    )
    if gold_standard is None:
        return None
    return gold_standard.answer

class PlaceboDescription(Recipe):
    agent_str: str

    def make_prompt(self, paper: Paper, experiment: str) -> str:
        raise NotImplementedError()

    async def get_gold_experiments(self, paper: Paper) -> list[str]:
        paper_id = get_full_document_id(paper.document_id)
        experiments: list[str] = list_experiments(document_id=paper_id)
        return experiments

    async def placebo_for_experiment(
        self, paper: Paper, experiment: str, record=recorder
    ) -> str:
        # Generate the QA prompt
        qa_prompt = self.make_prompt(paper, experiment)

        # Ask the agent to answer the prompt
        placebo_description = await self.agent(self.agent_str).complete(
            prompt=qa_prompt, max_tokens=300
        )

        # Save to trace
        evaluation = await EvaluateResult(mode=self.mode).run(
            gold_result=get_gold_placebo(paper.document_id, experiment),
            model_result=placebo_description,
            question=f'What was the placebo for the "{experiment}" experiment?',
        )
        record(evaluation=evaluation)

        return placebo_description

    async def run(self, paper: Paper, record=recorder):
        # Get the list of experiments (for now from gold standards)
        experiments = await self.get_gold_experiments(paper)

        results = {}
        for experiment in experiments:
            gold_placebo = get_gold_placebo(paper.document_id, experiment)
            if not gold_placebo or gold_placebo in (
                "No placebo",
                "Not mentioned",
                "Unclear",
            ):
                continue

            placebo = await self.placebo_for_experiment(paper, experiment)

            # Save results for each experiment
            recipe_results = [
                create_recipe_result(paper.document_id, experiment, placebo)
            ]
            self.maybe_add_to_results(recipe_results)

            results[experiment] = placebo

        return results

class PlaceboDescriptionInstruct(PlaceboDescription):
    name = "PlaceboDescriptionInstruct"
    agent_str = "instruct"

    def make_prompt(self, paper: Paper, experiment: str) -> str:
        paper_text = truncate_by_tokens(str(paper), max_tokens=3500)
        return f"""Text of a research paper:

{paper_text}

Question: In this paper, what was the placebo in the "{experiment}" experiment?

Answer:"""


import asyncio
import os
import subprocess
import sys

from PyQt5.QtGui import *
from PyQt5.QtWidgets import *

from global_modules import logs
from global_modules.macro_manager import reload_all

class Tray:
    def __init__(self, loop: asyncio.AbstractEventLoop):
        self.__loop = loop

        self.__app = QApplication([])
        self.__app.setQuitOnLastWindowClosed(False)
        self.__parent = QWidget()
        self.__icon = QIcon("tray_image/tray_enabled.png")

        self.tray = QSystemTrayIcon(self.__icon, self.__parent)
        self.tray.setToolTip("PyMacro")

        self.enabled = True

        self.__build_menu(enabled=True)

        self.tray.setVisible(True)

    def __build_menu(self, enabled: bool = True):
        menu = QMenu(self.__parent)

        item = QAction(self.__parent)
        item.setText("PyMacro - By picasso2005")
        item.setEnabled(False)
        menu.addAction(item)

        item = QAction(self.__parent)
        item.setSeparator(True)
        menu.addAction(item)

        if enabled:
            item = QAction(self.__parent)
            item.setText("Disable")
            item.triggered.connect(lambda: self.__toggle_enabled())

        else:
            item = QAction(self.__parent)
            item.setText("Enable")
            item.triggered.connect(lambda: self.__toggle_enabled())

        menu.addAction(item)

        item = QAction(self.__parent)
        item.setText("Reload registered macros")
        item.triggered.connect(lambda: reload_all())
        menu.addAction(item)

        item = QAction(self.__parent)
        item.setSeparator(True)
        menu.addAction(item)

        item = QAction(self.__parent)
        item.setText("Open macros folder")
        item.triggered.connect(lambda: self.__open_folder())
        menu.addAction(item)

        item = QAction(self.__parent)
        item.setText("Open logs")
        item.triggered.connect(lambda: self.__open_logs())
        menu.addAction(item)

        item = QAction(self.__parent)
        item.setSeparator(True)
        menu.addAction(item)

        item = QAction(self.__parent)
        item.setText("Exit")
        item.triggered.connect(lambda: self.__exit())
        menu.addAction(item)

        self.tray.setContextMenu(menu)

    def run_tray(self):
        self.__app.exec_()

    def __toggle_enabled(self):
        if self.enabled:
            self.enabled = False
            self.tray.setIcon(QIcon("tray_image/tray_disabled.png"))
            self.__build_menu(enabled=False)

            logs.info("tray", "Macros disabled")

        else:
            self.enabled = True
            self.tray.setIcon(QIcon("tray_image/tray_enabled.png"))
            self.__build_menu(enabled=True)

            logs.info("tray", "Macros enabled")

    @staticmethod
    def __open_logs():
        if sys.platform == "win32":
            subprocess.Popen(f"\"{os.getcwd()}\\{logs.LOG_PATH}\"", shell=True)

        elif sys.platform == "linux" or sys.platform == "linux2":
            subprocess.Popen(f"xdg-open \"{os.getcwd()}/{logs.LOG_PATH}\"", shell=True)

    @staticmethod
    def __open_folder():
        if sys.platform == "win32":
            subprocess.Popen(f"explorer \"{os.getcwd()}\\macros\"", shell=True)

        elif sys.platform == "linux" or sys.platform == "linux2":
            subprocess.Popen(f"xdg-open \"{os.getcwd()}/macros\"", shell=True)

    def __exit(self):
        self.__app.exit(0)
        self.__loop.stop()

        logs.info("tray", "Exiting app")

        exit(0)


import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from mlxtend.frequent_patterns import apriori
from mlxtend.frequent_patterns import association_rules

#Loading the dataset
orders= pd.read_csv('Instacart//orders.csv')
orderproducts= pd.read_csv('Instacart//order_products__train.csv')
products= pd.read_csv('Instacart//products.csv')
aisles= pd.read_csv('Instacart//aisles.csv')
departments= pd.read_csv('Instacart//departments.csv')

#Displays the column of orders dataset
orders.columns
#Displays the column of orders_products dataset
orderproducts.columns
#Displays the column of products dataset
products.columns
#Displays the column of aisles dataset
aisles.columns
#Displays the column of departments dataset
departments.columns

#Merges Orders dataset with orderproducts using order_id column
data = orders.merge(orderproducts, how='inner', on='order_id')
data.shape
#Merges new dataset dataset with products using product id
data = data.merge(products,how='inner', on='product_id')
data.shape
#Merges new dataset with aisles usimg aisle id
data = data.merge(aisles, how='inner', on='aisle_id')
data.shape
#Merges new dataset with departments using departent id
data = data.merge(departments, how='inner', on='department_id')
#Shape and First Five rows of merged dataframe
data.shape
data.head()

#Null Values
data.isnull().sum()

#grouping the dataset based on product id and  product name and aggregates with count
product_freq=data.groupby(['product_id','product_name']).agg({'product_name':'count'})
#Displays the number of time each product is ordered
product_freq
#Grouping the data by department id and counts the orderid in each department
dept_order=data.groupby('department_id').agg({'order_id':'count'})
dept_order.head(5)

#bar plot
plt.figure(figsize=(18,8))
sns.barplot(dept_order.index,dept_order.order_id);
plt.xlabel("Department ID")
plt.ylabel('Number of Orders')
plt.show();

#Groups the data by order day and counts the order id
weekday=data.groupby('order_dow').agg({'order_id':'count'})

#bar plot
plt.figure(figsize=(15,8))
sns.barplot(weekday.index,weekday.order_id);
plt.xlabel('Day of Week')
plt.ylabel('Number of Orders')
plt.show()

#Groups the data by order hour and counts the order id
hour_order=data.groupby('order_hour_of_day').agg({'order_id':'count'})

#bar plot
plt.figure(figsize=(15,8))
sns.barplot(hour_order.index,hour_order.order_id);
plt.xlabel('Hour')
plt.ylabel('Number of Orders')
plt.show()

order_reorder = data.groupby('reordered')['reordered'].aggregate('count')
order_reorder

plt.pie(order_reorder, labels=order_reorder.index,autopct='%1.1f%%', startangle=90);
plt.title('Reordered vs Not-Reordered');

hmdata=heatmapp.pivot('order_dow', 'order_hour_of_day', 'reordered')
plt.figure(figsize=(18,8))
sns.heatmap(hmdata);
plt.title('Order Day vs Order Hour vs Reorder')
plt.xlabel('Hour of the day')
plt.ylabel('Day of the week');

#Creating Basket
order_freq = data.drop(['user_id', 'eval_set', 'order_dow', 'order_hour_of_day', 'days_since_prior_order', 'add_to_cart_order', 'reordered','aisle_id', 'department_id', 'aisle', 'department'], axis=1)
mostfreq_products = order_freq.groupby('product_name').agg({'product_id':'count'}).sort_values(by='product_id', ascending=False).head(100).reset_index()
mostfreq_dataset = data[data['product_name'].isin(mostfreq_products['product_name'])]
basket = mostfreq_dataset.pivot_table(index='order_id',columns='product_name',values='reordered', aggfunc=np.sum, fill_value=0)
basket

def hot_encode(x):
    if(x<= 0):
        return 0
    if(x>= 1):
        return 1

basket = basket.applymap(hot_encode)
basket

freq_items = apriori(basket, min_support = 0.1, use_colnames = True)
freq_items
mba_rules = association_rules(freq_items, metric ="lift", min_threshold = 1.0)
mba_rules = rules.sort_values(['confidence', 'lift'], ascending =[False, False])


"""Start Bot"""
import logging

from butler_G.core import start_bot
from butler_G.constants import DEV_MODE

if __name__ == "__main__":
    logger = logging.getLogger()
    shndlr, *_ = logger.handlers
    shndlr.setLevel(logging.INFO)
    shndlr.setFormatter(
        logging.Formatter(
            "[{}] %(asctime)s - %(levelname)s : %(message)s".format(
                "DEV" if DEV_MODE else "PRD"
            )
        )
    )

    start_bot()


import keras
from keras.models import Sequential
from keras.preprocessing.image import ImageDataGenerator
from keras.datasets import cifar10
from keras.regularizers import l2
from keras.layers import Dense, Dropout, Activation, BatchNormalization, Conv2D, MaxPooling2D, Flatten, AveragePooling2D
from keras.initializers import he_normal
from keras.utils import plot_model
from keras import backend as K
#from utils import save_obj, load_obj

import numpy as np

import matplotlib.pyplot as plt
plt.ion()

import tensorflow as tf
# Set Memory allocation in tf/keras to Growth
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
K.set_session(sess)

# Plot callback
class Plot_Callback(keras.callbacks.Callback):
    def on_train_begin(self, logs={}):
        self.losses = []
        self.accuracies = []
        self.val_losses = []
        self.val_accuracies = []
        return

    def on_train_end(self, logs={}):
        plt.ioff()
        plt.show()

    def on_epoch_end(self, epoch, logs={}):
        self.losses.append(logs.get('loss'))
        self.accuracies.append(logs.get('acc'))
        self.val_losses.append(logs.get('val_loss'))
        self.val_accuracies.append(logs.get('val_acc'))

        ax = plt.gca()
        ax.clear()
        ax.plot(range(1,len(self.losses)+1),self.losses,color='r',label='train_loss')
        ax.plot(range(1,len(self.losses)+1),self.val_losses,color='b',label='val_loss')
        ax.set_ylim([0,1])
        ax.legend(loc=2)
        ax.set_xlabel('Epoch')
        ax.set_ylabel('Loss')
        plt.draw()
        plt.pause(0.01)
        return

if __name__ == "__main__":
    # Params
    use_datagen = True
    batch_size = 128
    epochs = 20

    # Load Data
    (x_train, y_train), (x_test, y_test) = cifar10.load_data()
    num_classes = int(max(y_train)) + 1
    x_train = x_train.astype('float32')/255
    x_test = x_test.astype('float32')/255
    y_train = keras.utils.to_categorical(y_train, num_classes)
    y_test = keras.utils.to_categorical(y_test, num_classes)

    model = Sequential()

    model.add(Conv2D(32, (3, 3), padding='same', use_bias=False, input_shape=x_train.shape[1:]))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(Conv2D(32, (3, 3), padding='same', use_bias=False))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    #model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.25))

    model.add(Conv2D(64, (3, 3), strides=(2,2), padding='same', use_bias=False))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(Conv2D(64, (3, 3), padding='same', use_bias=False))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    #model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.25))

    model.add(Conv2D(128, (3, 3), strides=(2,2), padding='same', use_bias=False))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(Conv2D(128, (3, 3), padding='same', use_bias=False))
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.25))

#==============================================================================
#     model.add(AveragePooling2D(pool_size=(8,8), strides=(1,1)))
#==============================================================================

    model.add(Flatten())

    model.add(Dense(512, use_bias=False))
    model.add(Activation('relu'))
    model.add(BatchNormalization())
    model.add(Dropout(0.5))

    model.add(Dense(num_classes, use_bias=False))
    model.add(BatchNormalization())
    model.add(Activation('softmax'))

#==============================================================================
#     model = Sequential()
#
#     model.add(Conv2D(32, (3, 3), padding='same', use_bias=False, input_shape=x_train.shape[1:]))
#     model.add(BatchNormalization())
#     model.add(Activation('relu'))
#     model.add(Conv2D(32, (3, 3), padding='same', use_bias=False))
#     model.add(BatchNormalization())
#     model.add(Activation('relu'))
#     model.add(MaxPooling2D(pool_size=(2, 2)))
#     model.add(Dropout(0.25))
#
#     model.add(Conv2D(64, (3, 3), padding='same', use_bias=False))
#     model.add(BatchNormalization())
#     model.add(Activation('relu'))
#     model.add(Conv2D(64, (3, 3), padding='same', use_bias=False))
#     model.add(BatchNormalization())
#     model.add(Activation('relu'))
#     model.add(MaxPooling2D(pool_size=(2, 2)))
#     model.add(Dropout(0.25))
#
#     model.add(Conv2D(128, (3, 3), padding='same', use_bias=False))
#     model.add(BatchNormalization())
#     model.add(Activation('relu'))
#     model.add(Conv2D(128, (3, 3), padding='same', use_bias=False))
#     model.add(BatchNormalization())
#     model.add(Activation('relu'))
#     model.add(MaxPooling2D(pool_size=(2, 2)))
#     model.add(Dropout(0.25))
#
#     model.add(AveragePooling2D(pool_size=(4,4), strides=(1,1)))
#
#     model.add(Flatten())
#
# #==============================================================================
# #     model.add(Dense(512, use_bias=False))
# #     model.add(Activation('relu'))
# #     model.add(BatchNormalization())
# #     model.add(Dropout(0.5))
# #==============================================================================
#
#     model.add(Dense(num_classes, use_bias=False))
#     model.add(BatchNormalization())
#     model.add(Activation('softmax'))
#==============================================================================

    # Compile model with optimizer
    #opt = keras.optimizers.Adam(lr=0.01, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)
    #opt = keras.optimizers.SGD(lr=0.1, momentum=0.9, nesterov=True)
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])

    #Print/Plot Model
    model.summary()
    plot_model(model, show_shapes=True, show_layer_names=False)

    # Fit
    plot = Plot_Callback()
    if use_datagen:
        datagen = ImageDataGenerator(
               width_shift_range=0.1,
               height_shift_range=0.1,
               #zoom_range=0.2,
               horizontal_flip=True)
        datagen.fit(x_train)
        model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size), steps_per_epoch=x_train.shape[0] // batch_size, epochs=epochs, validation_data=(x_test, y_test), callbacks=[plot])
    else:
        model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_data=(x_test, y_test), shuffle=True, callbacks=[plot])