import subprocess
import argparse
import sys
import os
from os import path
import pandas as pd
import tempfile
from atomicwrites import atomic_write
import logging

_ROOT = path.abspath(path.join(os.getcwd(), "."))

logger = logging.getLogger('GMSC-mapper')

def parse_args(args):
    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter,
                                     description='GMSC-mapper')
    subparsers = parser.add_subparsers(title='GMSC-mapper subcommands',
                                       dest='cmd',
                                       metavar='')

    cmd_download_db = subparsers.add_parser('downloaddb',
                                          help='Download target database')

    cmd_download_db.add_argument('--dbdir',
                               required=False,
                               help='Path to the database files.',
                               dest='dbdir',
                               default = path.join(_ROOT, 'db'))
    cmd_download_db.add_argument('--all', action="store_true", dest='all',
                                help='Download all database')
    cmd_download_db.add_argument('-f', action="store_true", dest='force',
                                help='Force download even if the files exist')

    cmd_create_db = subparsers.add_parser('createdb',
                                          help='Create target database')
    cmd_create_db.add_argument('-i',
                               required=True,
                               help='Path to the GMSC FASTA file.',
                               dest='target_faa',
                               default = None)
    cmd_create_db.add_argument('-o', '--output',
                               required=False,
                               help='Path to database output directory.',
                               dest='output',
                               default = path.join(_ROOT, 'db'))
    cmd_create_db.add_argument('-m', '--mode',
                               required=True,
                               help='Alignment tool (Diamond / MMseqs2)',
                               dest='mode',
                               default = None)
    cmd_create_db.add_argument('--quiet',
                                action='store_true',
                                dest='quiet',
                                help='Disable alignment console output')

    parser.add_argument('-i', '--input',
                        required=False,
                        help='Path to the input genome contig sequence FASTA file.',
                        dest='genome_fasta',
                        default = None)

    parser.add_argument('--nt-genes', '--nt_genes',
                        required=False,
                        help='Path to the input nucleotide gene sequence FASTA file.',
                        dest='nt_input',
                        default=None)

    parser.add_argument('--aa-genes', '--aa_genes',
                        required=False,
                        help='Path to the input amino acid sequence FASTA file.',
                        dest='aa_input',
                        default=None)

    parser.add_argument('-o', '--output',
                        required=False,
                        help='Output directory (will be created if non-existent)',
                        dest='output',
                        default=path.join(_ROOT, 'output'))

    parser.add_argument('--tool', '--tool',
                        required=False,
                        choices=['diamond', 'mmseqs'],
                        help='Sequence alignment tool (Diamond / MMseqs).',
                        dest='tool',
                        default='diamond')

    parser.add_argument('-s', '--sensitivity',
                        required=False,
                        help='Sensitivity.',
                        dest='sensitivity')

    parser.add_argument('--id', '--id',
                        required=False,
                        help='Minimum identity to report an alignment (range 0.0-1.0).',
                        dest='identity',
                        default=0.0)

    parser.add_argument('--cov', '--cov',
                        required=False,
                        help='Minimum coverage to report an alignment (range 0.0-1.0).',
                        dest='coverage',
                        default=0.9)

    parser.add_argument('-e', '--evalue',
                        required=False,
                        help='Maximum e-value to report alignments.',
                        dest='evalue',
                        default=0.00001)

    parser.add_argument('-t', '--threads',
                        required=False,
                        help='Number of CPU threads.',
                        dest='threads',
                        default=1)

    parser.add_argument('--filter','--filter',action='store_true', help='Use this to filter <100 aa or <303 nt input sequences.')

    parser.add_argument('--no-habitat','--no-habitat',action='store_true', dest='nohabitat', help='Use this if no need to annotate habitat')

    parser.add_argument('--no-taxonomy', '--no-taxonomy',action='store_true', dest='notaxonomy', help='Use this if no need to annotate taxonomy')

    parser.add_argument('--no-quality', '--no-quality',action='store_true', dest='noquality', help='Use this if no need to annotate quality')

    parser.add_argument('--no-domain', '--no-domain',action='store_true', dest='nodomain', help='Use this if no need to annotate quality')

    parser.add_argument('--quiet','--quiet',action='store_true', help='Disable alignment console output')

    parser.add_argument('--dbdir', '--dbdir',
                        required=False,
                        help='Path to the GMSC database directory.',
                        dest='dbdir',
                        default = path.join(_ROOT, 'db'))

    return parser.parse_args(args[1:])

def check_install():
    from shutil import which
    import sys

    has_mmseqs = False
    has_diamond = False
    dependencies = ['diamond', 'mmseqs']
    logger.debug("Looking for dependencies...")

    for dep in dependencies:
        p = which(dep)
        if p:
            if dep == 'diamond':
                has_diamond = True
            if dep == 'mmseqs':
                has_mmseqs = True

    if not has_diamond and not has_mmseqs:
        logger.warning('GMSC-mapper Warning: Neither diamond nor mmseqs appear to be available! It will download diamond and mmseqs.\n')
        subprocess.check_call(['wget','https://mmseqs.com/latest/mmseqs-linux-avx2.tar.gz',
                                '-P','./bin'])
        subprocess.check_call(['tar','xvfz',
                                './bin/mmseqs-linux-avx2.tar.gz',
                                '-C','./bin'])
        subprocess.check_call(['wget','http://github.com/bbuchfink/diamond/releases/download/v2.1.8/diamond-linux64.tar.gz',
                                '-P','./bin'])
        subprocess.check_call(['tar','xvfz',
                                './bin/diamond-linux64.tar.gz',
                                '-C','./bin'])
    else:
        logger.info('Dependencies installation is OK\n')
    return has_diamond,has_mmseqs

def validate_args(args,has_diamond,has_mmseqs):
    def expect_file(f):
        if not os.path.exists(f):
            sys.stderr.write(f"GMSC-mapper Error: Expected file '{f}' does not exist\n")
            sys.exit(1)

    def expect_database(f):
        if not os.path.exists(f):
            sys.stderr.write(f"GMSC-mapper Error: Expected file '{f}' does not exist. Please use --dbdir to assign your database directory.\n")
            sys.exit(1)

    if not args.genome_fasta and not args.aa_input and not args.nt_input:
        pass
    elif args.genome_fasta and not args.aa_input and not args.nt_input:
        expect_file(args.genome_fasta)
    elif args.aa_input and not args.genome_fasta and not args.nt_input:
        expect_file(args.aa_input)
    elif args.nt_input and not args.genome_fasta and not args.aa_input:
        expect_file(args.nt_input)
    else:
        sys.stderr.write("GMSC-mapper Error: --input or --aa-genes or --nt_genes shouldn't be assigned at the same time\n")
        sys.exit(1)

    if args.tool == "diamond" and not has_diamond:
        logger.warning("GMSC-mapper Warning:mmseqs is not available.It will download diamond.\n")
        subprocess.check_call(['wget','http://github.com/bbuchfink/diamond/releases/download/v2.1.8/diamond-linux64.tar.gz',
                                '-P','./bin'])
        subprocess.check_call(['tar','xvfz',
                                './bin/diamond-linux64.tar.gz',
                                '-C','./bin'])
    if args.tool == "mmseqs" and not has_mmseqs:
        logger.warning("GMSC-mapper Warning:mmseqs is not available.It will download mmseqs.\n")
        subprocess.check_call(['wget','https://mmseqs.com/latest/mmseqs-linux-avx2.tar.gz',
                                '-P','./bin'])
        subprocess.check_call(['tar','xvfz',
                                './bin/mmseqs-linux-avx2.tar.gz',
                                '-C','./bin'])

    if args.tool == "diamond":
        expect_database(path.join(args.dbdir, "diamond_targetdb.dmnd"))
    if args.tool == "mmseqs":
        expect_database(path.join(args.dbdir, "mmseqs_targetdb"))

    if not args.nohabitat:
        expect_database(path.join(args.dbdir, "GMSC10.90AA.habitat.index.tsv"))
        expect_database(path.join(args.dbdir, "GMSC10.90AA.habitat.npy"))

    if not args.notaxonomy:
        expect_database(path.join(args.dbdir, "GMSC10.90AA.taxonomy.index.tsv"))
        expect_database(path.join(args.dbdir, "GMSC10.90AA.taxonomy.npy"))

    if not args.noquality:
        expect_database(path.join(args.dbdir, "GMSC10.90AA.high_quality.tsv.xz"))

    if not args.nodomain:
        expect_database(path.join(args.dbdir, "GMSC10.90AA.cdd.tsv.xz"))

def download_db(args):
    from gmsc_mapper.utils import ask

    if args.force or not os.path.exists(os.path.join(args.dbdir,'GMSC10.90AA.faa.gz')):
        if args.all or ask("Download 90AA fasta file (~11G)?") == 'y':
            logger.info('Start downloading 90AA fasta file...')
            subprocess.check_call(['wget','https://gmsc-api.big-data-biology.org/files/GMSC10.90AA.faa.gz',
                                '-P',args.dbdir])
            logger.info('90AA fasta file has been downloaded successfully.')
        else:
            print('Skip downloading 90AA fasta file.')
    else:
        print('90AA fasta file already exists. Skip downloading 90AA fasta file. Use -f to force download')

    if args.force or not os.path.exists(os.path.join(args.dbdir,'GMSC10.90AA.habitat.npy')) or not os.path.exists(os.path.join(args.dbdir,'GMSC10.90AA.habitat.index.tsv')):
        if args.all or ask("Download habitat index file (~2.3G)?") == 'y':
            logger.info('Start downloading habitat index file...')
            subprocess.check_call(['wget','https://gmsc-api.big-data-biology.org/files/GMSC10.90AA.habitat.npy',
                                '-P',args.dbdir])
            subprocess.check_call(['wget','https://gmsc-api.big-data-biology.org/files/GMSC10.90AA.habitat.index.tsv',
                                '-P',args.dbdir])
            logger.info('Habitat index file has been downloaded successfully.')
        else:
            print('Skip downloading habitat index file.')
    else:
        print('Habitat index file already exists. Skip downloading habitat index file. Use -f to force download')

    if args.force or not os.path.exists(os.path.join(args.dbdir,'GMSC10.90AA.taxonomy.npy')) or not os.path.exists(os.path.join(args.dbdir,'GMSC10.90AA.taxonomy.index.tsv')):
        if args.all or ask("Download taxonomy index file (~2.3G)?") == 'y':
            logger.info('Start downloading taxonomy index file...')
            subprocess.check_call(['wget','https://gmsc-api.big-data-biology.org/files/GMSC10.90AA.taxonomy.npy',
                                '-P',args.dbdir])
            subprocess.check_call(['wget','https://gmsc-api.big-data-biology.org/files/GMSC10.90AA.taxonomy.index.tsv',
                                '-P',args.dbdir])
            logger.info('Taxonomy index file has been downloaded successfully.')
        else:
            print('Skip downloading taxonomy index file.')
    else:
        print('Taxonomy index file already exists. Skip downloading taxonomy index file. Use -f to force download')

    if args.force or not os.path.exists(os.path.join(args.dbdir,'GMSC10.90AA.high_quality.tsv.xz')):
        if args.all or ask("Download quality index file (2.6M)?") == 'y':
            logger.info('Start downloading quality index file...')
            subprocess.check_call(['wget','https://gmsc-api.big-data-biology.org/files/GMSC10.90AA.high_quality.tsv.xz',
                                '-P',args.dbdir])
            logger.info('Quality index file has been downloaded successfully.')
        else:
            print('Skip downloading quality index file.')
    else:
        print('Quality index file already exists. Skip downloading quality index file. Use -f to force download')

    if args.force or not os.path.exists(os.path.join(args.dbdir,'GMSC10.90AA.cdd.tsv.xz')):
        if args.all or ask("Download conserved domain index file (88M)?") == 'y':
            logger.info('Start downloading conserved domain index file...')
            subprocess.check_call(['wget','https://gmsc-api.big-data-biology.org/files/GMSC10.90AA.cdd.tsv.xz',
                                '-P',args.dbdir])
            logger.info('Conserved domain index file has been downloaded successfully.')
        else:
            print('Skip downloading conserved domain index file.')
    else:
        print('Conserved domain index file already exists. Skip downloading conserved domain index file. Use -f to force download')

def create_db(args):
    from shutil import which

    if not os.path.exists(args.output):
        os.makedirs(args.output)

    diamond_out_db = path.join(args.output,"diamond_targetdb")
    mmseqs_out_db = path.join(args.output,"mmseqs_targetdb")

    if which('diamond'):
        diamond = 'diamond'
    else:
        diamond = './bin/diamond'

    if which('mmseqs'):
        mmseqs = 'mmseqs'
    else:
        mmseqs = './bin/mmseqs/bin/mmseqs'

    if args.quiet:
        diamond_cmd = [diamond,'makedb',
                        '--in',args.target_faa,
                        '-d',diamond_out_db,
                        '--quiet']
        mmseqs_cmd = [mmseqs,'createdb',
                        args.target_faa,
                        mmseqs_out_db,
                        '-v','0']
    else:
        diamond_cmd = [diamond,'makedb',
                    '--in',args.target_faa,
                    '-d',diamond_out_db]
        mmseqs_cmd = [mmseqs,'createdb',
                    args.target_faa,
                    mmseqs_out_db]

    if args.mode == "diamond":
        logger.info('Start creating Diamond database...')
        subprocess.check_call(diamond_cmd)
        logger.info('Diamond database has been created successfully.')

    if args.mode == "mmseqs":
        logger.debug('Start creating MMseqs database...')
        subprocess.check_call(mmseqs_cmd)
        logger.info('MMseqs database has been created successfully.')

def predict(args,tmpdir):
    from gmsc_mapper.predict import predict_genes,filter_smorfs

    logger.debug('Starting smORF prediction...')

    predicted_smorf = path.join(tmpdir,"predicted.smorf.faa")
    filtered_smorf = path.join(args.output,"predicted.filterd.smorf.faa")

    predict_genes(args.genome_fasta,predicted_smorf)
    if not path.getsize(predicted_smorf):
        logger.info("GMSC-mapper Info:No smORFs have been predicted.Please check your input file.\n")
        return ("",False)
    else:
        filter_smorfs(predicted_smorf, filtered_smorf)
        if not path.getsize(filtered_smorf):
            logger.info("GMSC-mapper Info:No smORFs remained after filtering by length(<100aa).\n")
            return ("",False)
        else:
            logger.info('smORF prediction complete')
            return (filtered_smorf,True)

def translate_gene(args,tmpdir):
    from gmsc_mapper.translate import translate_gene
    logger.debug('Starting gene translation...')
    translated_file = translate_gene(args.nt_input,tmpdir)
    logger.info('Gene translation complete')
    return translated_file

def check_length(queryfile):
    from gmsc_mapper.fasta import fasta_iter
    logger.debug('Start length check...')
    if all(len(seq) < 303
                for _, seq in fasta_iter(queryfile)):
        logger.warning('GMSC-mapper Warning: Input sequences are all greater than 300bps.\n'
                       'Please check if your input consists of contigs, which should use -i not --nt-genes or --aa-genes as input. '
                       'However, we will regard your input sequences as nucleotide genes and continue to process.\n')

    logger.info('Length checking has done.\n')

def filter_length(queryfile,tmpdir,N):
    from gmsc_mapper.filter_length import filter_length
    logger.debug('Starting length filter...')
    filtered_file = filter_length(queryfile,tmpdir,N)
    logger.info('Length filter complete')
    return filtered_file

def mapdb_diamond(args,queryfile):
    from shutil import which

    logger.debug('Starting smORF mapping...')

    resultfile = path.join(args.output,"alignment.out.smorfs.tsv")
    outfmt = '6,qseqid,sseqid,full_qseq,full_sseq,qlen,slen,length,qstart,qend,sstart,send,bitscore,pident,evalue,qcovhsp,scovhsp'

    if which('diamond'):
        diamond = 'diamond'
    else:
        diamond = './bin/diamond'

    database = path.join(args.dbdir, "diamond_targetdb.dmnd")

    if args.sensitivity != '--default':
        diamond_cmd = [diamond,'blastp',
                    '-q',queryfile,
                    '-d',database,
                    '-o',resultfile,
                    args.sensitivity,
                    '-e',str(args.evalue),
                    '--id',str(float(args.identity)*100),
                    '--query-cover',str(float(args.coverage)*100),
                    '--subject-cover',str(float(args.coverage)*100),
                    '-p',str(args.threads),
                    '--outfmt'] + outfmt.split(',')
    else:
        diamond_cmd = [diamond,'blastp',
                    '-q',queryfile,
                    '-d',database,
                    '-o',resultfile,
                    '-e',str(args.evalue),
                    '--id',str(float(args.identity)*100),                                                                                '--query-cover',str(float(args.coverage)*100),
                    '--subject-cover',str(float(args.coverage)*100),                                                                        '-p',str(args.threads),                                                                                '--outfmt'] + outfmt.split(',')
    if args.quiet:
        diamond_cmd.append('--quiet')

    subprocess.check_call(diamond_cmd)

    logger.info('smORF mapping complete')
    return resultfile

def mapdb_mmseqs(args, queryfile, tmpdir):
    from shutil import which

    logger.info('Start smORF mapping...')

    querydb = path.join(tmpdir,"query.db")
    resultdb = path.join(tmpdir,"result.db")
    tmp = path.join(tmpdir,"tmp","")
    resultfile = path.join(args.output,"alignment.out.smorfs.tsv")
    outfmt = 'query,target,qseq,tseq,qlen,tlen,alnlen,qstart,qend,tstart,tend,bits,pident,evalue,qcov,tcov'

    if which('mmseqs'):
        mmseqs = 'mmseqs'
    else:
        mmseqs = './bin/mmseqs/bin/mmseqs'

    database = path.join(args.dbdir, "mmseqs_targetdb")

    mmseqs_cmd_db = [mmseqs, 'createdb', queryfile, querydb]
    mmseqs_cmd_search = [mmseqs,'search',
                        querydb,
                        database,
                        resultdb,
                        tmp,
                        '-s',str(args.sensitivity),
                        '-e',str(args.evalue),
                        '--min-seq-id',str(args.identity),
                        '-c',str(args.coverage),
                        '--threads',str(args.threads)]
    mmseqs_cmd_out = [mmseqs,'convertalis',
                    querydb,
                    database,
                    resultdb,
                    resultfile,
                    '--format-output', outfmt]
    for mmseqs_cmd in [mmseqs_cmd_db,mmseqs_cmd_search,mmseqs_cmd_out]:
        if args.quiet:
            mmseqs_cmd.extend(['-v', '0'])
        subprocess.check_call(mmseqs_cmd)

    logger.info('smORF mapping complete')
    return resultfile

def generate_fasta(output,queryfile,resultfile):
    import pandas as pd
    from gmsc_mapper.fasta import fasta_iter

    try:
        result = pd.read_csv(resultfile, sep='\t',header=None)
    except:
        print('GMSC-mapper info: There is no alignment results between your input sequences and GMSC.\n')
        logger.info('GMSC-mapper info: There is no alignment results between your input sequences and GMSC.\n')
        return ("",False)
    else:
        logger.debug('Start smORF fasta file generation...')
        fastafile = path.join(output,"mapped.smorfs.faa")

        smorf_id = set(result.iloc[:, 0].tolist())

        with open(fastafile,"wt") as f:
            for ID,seq in fasta_iter(queryfile):
                if ID in smorf_id:
                    f.write(f'>{ID}\n{seq}\n')
        logger.debug('smORF fasta file generation complete')
        return (fastafile,True)

def habitat(args,resultfile):
    from gmsc_mapper.map_habitat import smorf_habitat
    logger.debug('Starting habitat annotation...')

    habitatindex = path.join(args.dbdir, "GMSC10.90AA.habitat.index.tsv")
    habitat = path.join(args.dbdir, "GMSC10.90AA.habitat.npy")
    r_habitat = smorf_habitat(habitatindex,args.output,habitat,resultfile)

    logger.info('habitat annotation has done.')
    return r_habitat

def taxonomy(args,resultfile,tmpdirname):
    from gmsc_mapper.map_taxonomy import deep_lca,taxa_summary
    logger.debug('Start taxonomy annotation...')

    taxonomyindex = path.join(args.dbdir, "GMSC10.90AA.taxonomy.index.tsv")
    taxonomy = path.join(args.dbdir, "GMSC10.90AA.taxonomy.npy")
    deep_lca(taxonomyindex,taxonomy,args.output,resultfile,tmpdirname)
    r_summary = taxa_summary(args.output)

    logger.info('Taxonomy annotation complete.')
    return r_summary

def quality(args,resultfile):
    from gmsc_mapper.map_quality import smorf_quality
    logger.debug('Start quality annotation...')

    quality = path.join(args.dbdir, "GMSC10.90AA.high_quality.tsv.xz")
    number,percentage = smorf_quality(args.output,quality,resultfile)

    logger.info('Quality annotation completed.')
    return number,percentage

def domain(args,resultfile):
    from gmsc_mapper.map_domain import smorf_domain
    logger.debug('Start domain annotation...')

    domain = path.join(args.dbdir, "GMSC10.90AA.cdd.tsv.xz")
    number = smorf_domain(domain,args.output,resultfile)
    logger.info('Domain annotation completed.')
    return number

def predicted_smorf_count(file_name):
    return sum(1 for _ in open(file_name, 'rt'))

def main(args=None):
    if args is None:
        args = sys.argv
    args = parse_args(args)
    if not args.cmd and not args.genome_fasta and not args.aa_input and not args.nt_input:
        sys.stderr.write("GMSC-mapper Error: Please see gmsc-mapper -h. Choose a subcommand or input file.\n")
        sys.exit(1)
    has_diamond,has_mmseqs = check_install()

    if args.cmd == 'createdb':
        create_db(args)

    if args.cmd == 'downloaddb':
        download_db(args)

    if not args.cmd:
        validate_args(args,has_diamond,has_mmseqs)

        if args.tool == 'diamond':
            args.sensitivity = {
                None: '--sensitive',
                '1': '--fast',
                '2': '--default',
                '3': '--mid-sensitive',
                '4': '--sensitive',
                '5': '--more-sensitive',
                '6': '--very-sensitive',
                '7': '--ultra-sensitive',
            }.get(args.sensitivity, args.sensitivity)

        if args.tool == 'mmseqs':
            if args.sensitivity is None:
                args.sensitivity = 5.7

        os.makedirs(args.output, exist_ok=True)

        with tempfile.TemporaryDirectory() as tmpdir:
            try:
                summary = []
                summary.append(f'# Total number')
                if args.genome_fasta:
                    (queryfile,ifpredict) = predict(args,tmpdir)
                    if ifpredict:
                        smorf_number = int(predicted_smorf_count(queryfile)/2)
                        summary.append(f'{smorf_number} smORFs are predicted in total.')
                    else:
                        summary.append(f'No smORFs are predicted.')
                if args.nt_input:
                    if args.filter:
                        args.nt_input = filter_length(args.nt_input,tmpdir,303)
                    else:
                        check_length(args.nt_input)
                    queryfile = translate_gene(args,tmpdir)
                if args.aa_input:
                    if args.filter:
                        args.aa_input = filter_length(args.aa_input,tmpdir,100)
                    queryfile = args.aa_input

                if (args.genome_fasta and ifpredict) or args.nt_input or args.aa_input:
                    if args.tool == 'diamond':
                        resultfile = mapdb_diamond(args,queryfile)
                    if args.tool == 'mmseqs':
                        resultfile = mapdb_mmseqs(args,queryfile,tmpdir)

                    (fastafile,ifsuccess) = generate_fasta(args.output,queryfile,resultfile)
                    if ifsuccess:
                        smorf_number = int(predicted_smorf_count(fastafile)/2)
                        summary.append(f'{smorf_number} smORFs are aligned against GMSC in total.\n')

                        if not args.noquality:
                            summary.append(f'# Quality')
                            number,percentage = quality(args,resultfile)
                            summary.append(f'{number} ({percentage:.2%}) aligned smORFs are high quality.\n')

                        if not args.nohabitat:
                            summary.append(f'# Habitat')
                            single_number,single_percentage,multi_number,multi_percentage = habitat(args,resultfile)
                            summary.append(f'{single_number} ({single_percentage:.2%}) aligned smORFs are single-habitat.')
                            summary.append(f'{multi_number} ({multi_percentage:.2%}) aligned smORFs are multi-habitat.\n')

                        if not args.notaxonomy:
                            summary.append(f'# Taxonomy')
                            annotated_number,rank_number,rank_percentage = taxonomy(args,resultfile,tmpdir)
                            summary.append(f'{annotated_number} ({1 - rank_percentage["no rank"]:.2%}) aligned smORFs have taxonomy annotation.')
                            for rank in ['kingdom','phylum','class','order','family','genus','species']:
                                summary.append(f'{rank_number[rank]} ({rank_percentage[rank]:.2%}) aligned smORFs are annotated at level of {rank}.')

                        if not args.nodomain:
                            summary.append(f'\n# Conserved domain')
                            number = domain(args,resultfile)
                            summary.append(f'{number} aligned smORFs are annotated with CDD database.\n')
                    else:
                        summary.append(f'None of sequences are aligned against GMSC.\n')

                with atomic_write(f'{args.output}/summary.txt', overwrite=True) as ofile:
                    for s in summary:
                        print(s)
                        ofile.write(f'{s}\n')

            except Exception as e:
                sys.stderr.write('GMGC-mapper Error: ')
                sys.stderr.write(str(e))
                sys.stderr.write('\n')
                sys.exit(1)

if __name__ == '__main__':
    main(sys.argv)


import mechanize as mechanize
from bs4 import BeautifulSoup as bs
import http.cookiejar as cookielib

import pandas as pd
from pandas.core.frame import DataFrame

import pandas as pd
from pandas.core.frame import DataFrame

class getGroup:
    def __name__(self):
        return 'getGroup'

    def __init__(self):
        # get col name value Código Disciplina,Turma,Professor,Link of csv file
        self.filename = "sigaa_server/CataGrupoUnifei.csv"
        self.col_name = ['codigo', 'turma', 'professor', 'wpp_url']
        self.index = 0
        pass

    def get_group_csv(self):
        # get csv file(
        pd_csv = pd.read_csv(self.filename, names=self.col_name, header=1, index_col=False).fillna('')
        json = pd_csv.to_dict('records')

        return json

class ScrapingSigaa():

    def __init__(self, userlogin, userpass, url='https://sigaa.unifei.edu.br/sigaa/verTelaLogin.do'):
        self.userlogin = userlogin
        self.userpass = userpass
        self.error = "Usuário e/ou senha inválidos";
        self.cookies = cookielib.CookieJar()  # cria um repositório de cookies
        self.browser = mechanize.Browser()    # inicia um browser
        self.browser.set_handle_robots(False)
        self.browser.set_cookiejar(self.cookies)   # aponta para o seu repositório de cookies
        self.url = url

        # inicializando grupos
        self.groups = getGroup().get_group_csv()

        #inicializando browser

        self.browser.open(self.url)
        self.browser.select_form(nr=0)
        self.browser.form['user.login'] = self.userlogin
        self.browser.form['user.senha'] = self.userpass
        self.browser.submit()
        self.pagina = self.browser.response().read()
        self.soup = bs(self.pagina, 'html.parser')
        self.isLoginFailed()
        self.isPageValid()

    def isLoginFailed(self):
        res = self.soup.find('center', style=lambda value: value and 'color: #922' in value)
        if(res != None):
            raise ValueError("Credencial inválida")

    def isPageValid(self):
        self.perfil = self.soup.find(id="perfil-docente")
        if(self.perfil == None):
            while(self.perfil == None):
                self.browser.select_form(nr=0)
                self.browser.submit()
                self.pagina = self.browser.response().read()
                self.soup = bs(self.pagina, 'html.parser')
                self.perfil = self.soup.find(id="perfil-docente")
            self.nome = self.perfil.find(name="b").text
        else:
            self.perfil = self.soup.find(id="perfil-docente")
            self.nome = self.perfil.find(name="b").text

    def dispose(self):
        self.browser.close()

    def changeHour(self, sigaaBase):
        DIAS = {
            2: 1,
            3: 2,
            4: 3,
            5: 4,
            6: 5,
            7: 6
        }

        HORARIOS = {
            'M1': {"inicio": '07:00', "fim": '07:55'},
            'M2': {"inicio": '07:55', "fim": '08:50'},
            'M3': {"inicio": '18:50', "fim": '09:45'},
            'M4': {"inicio": '10:10', "fim": '11:05'},
            'M5': {"inicio": '11:05', "fim": '12:00'},
            'T1': {"inicio": '13:30', "fim": '14:25'},
            'T2': {"inicio": '14:25', "fim": '15:20'},
            'T3': {"inicio": '15:45', "fim": '16:40'},
            'T4': {"inicio": '16:40', "fim": '17:35'},
            'T5': {"inicio": '17:35', "fim": '18:30'},
            'N1': {"inicio": '19:00', "fim": '19:50'},
            'N2': {"inicio": '19:50', "fim": '20:40'},
            'N3': {"inicio": '21:00', "fim": '21:50'},
            'N4': {"inicio": '21:50', "fim": '22:40'},
            'N5': {"inicio": '22:40', "fim": '23:30'}

        }

        TURNOS = ['M', 'T', 'N']

        saida = dict()

        for bases in sigaaBase.split(","):
            pos = 0

            horario = bases.strip().replace(")","").split(" (")

            dia = []
            inicio = []
            fim = []

            for tempos in horario[0].split(" "):
                lista = list(tempos)

                for turno in TURNOS:
                    if(turno in lista):
                        pos = lista.index(turno)

                dia.extend(DIAS[int(dias)] for dias in lista[:pos])
                inicio.append(HORARIOS[lista[pos]+lista[pos+1]]["inicio"])
                fim.append(HORARIOS[lista[pos]+lista[-1]]["fim"])

            saida = {
                "dias_semana": dia,
                "hr_inicio": inicio*len(dia) if len(dia) != len(inicio) else inicio,
                "hr_fim": fim*len(dia) if len(dia) != len(fim) else fim,
                }
        return saida

    def getDataUser(self):
        print("iniciando busca por user: ", self.userlogin)

        try:
            dados_universidade = self.perfil.find(name="table")

            i = 0
            saida = dict()
            saida["nome"] = self.nome

            for row in dados_universidade.findAll("td"):

                if(i == 1):
                    saida["matricula"] = int(row.text.strip())
                if(i == 3):
                    saida["curso"] = row.text.strip().replace("\n","").replace("\t","")
                if(i == 5):
                    saida["nivel"] = row.text.strip()
                if(i == 7):
                    saida["status"] = row.text.strip()
                if(i == 11):
                    saida["entrada"] = row.text.strip()
                if(i == 16):
                    saida["mc"] = float(row.text.strip())
                if(i == 18):
                    saida["ira"] = float(row.text.strip())
                if(i == 20):
                    saida["iech"] = float(row.text.strip())
                if(i == 22):
                    saida["iepl"] = float(row.text.strip())
                if(i == 24):
                    saida["iea"] = float(row.text.strip())
                if(i == 26):
                    saida["iechs"] = float(row.text.strip())
                if(i == 30):
                    saida["ch_obrigatoria_pendente"] = int(row.text.strip())
                if(i == 32):
                    saida["ch_optativa_pendente"] = int(row.text.strip())
                if(i == 34):
                    saida["ch_total_curriculo"] = int(row.text.strip())
                if(i == 36):
                    saida["ch_complementar_pendente"] = int(row.text.strip())
                i += 1

            return saida
        except Exception as e:
            return str(e)

    def getBodyClass(self):
        print("iniciando busca pelas atualizacoes das disciplinas")

        return ""

    def getNotices(self):
        print("iniciando busca por noticias")
        try:
            saida = []
            notices = self.soup.find(id="formAtualizacoesTurmas")
            for tables in notices.find_all('table'):
                s = {}
                i = 0
                for notice in tables.find_all('td'):
                    if(i%2==0):
                        s["title"] = notice.text.strip().replace("\t", "").replace("\n", "")
                    else:
                        s["notice"] = notice.text.strip().replace("\t", "").replace("\n", "")
                    i += 1
                saida.append(s)

            return saida
        except Exception as e:
            return str(e)

    def getTasks(self):
        print("iniciando busca por tarefas do user: ", self.userlogin)

        try:
            portal = self.soup.find(id="avaliacao-portal")
            vazio = portal.find(class_='vazio')

            if(vazio!=None):
                return vazio.text.strip()

            df = DataFrame()
            feitos = []
            tipos = []
            atividades = []
            disciplina = []

            for ativ in portal.find_all('td', style=lambda value: value and 'text-align:center' in value):
                if (ativ.find(name="img") != None) : feitos.append(ativ.find(name="img").get('title'))
                else: feitos.append("Atividade passada")

            for ativ in portal.find_all(name="small"):
                novo = []
                for x in ativ.text.split("\n"):
                    item = x
                    for y in ['\n', '\t', '/', '.', '-', '(', ')']:
                        item = item.replace(y, "")
                    item = item.split(":")
                    if (item != ['']): novo.extend(item)
                novo = [elemento for elemento in novo if elemento.strip()]
                atividades.append(novo[2].strip())
                tipos.append(novo[1])
                disciplina.append(novo[0])

            df_data = pd.read_html(str(portal), header=0)[0].iloc[:,1]
            df_data = df_data.tolist()[1:]

            for x in range(len(df_data)):
                if("(" in df_data[x]):
                    if(feitos[x] == "Atividade passada"):
                        feitos[x] = "Atividade para a próxima semana"

            df["data"] = df_data
            df["data"] = df["data"].apply(lambda x : pd.to_datetime(x.split(" ")[0], dayfirst=True))
            df["feito"] = feitos
            df["tipo"] = tipos
            df["atividade"] = atividades
            df["disciplina"] = disciplina

            df = df.sort_values(by="data")

            return df.to_dict('records')
        except Exception as e:
            return str(e)

    def getClasses(self):
        print("iniciando busca por aulas do user: ", self.userlogin)

        try:
            aulas = self.soup.find(id="turmas-portal")
            vazio = aulas.find(class_='vazio')

            if(vazio!=None):
                return vazio.text.strip()

            df = pd.read_html(str(aulas), header=0)[-1].iloc[1:,0:3].dropna()
            df.columns = ["disciplina", "local", "horario"]
            df["horario"] = df["horario"].apply(lambda x : self.changeHour(sigaaBase=x))
            return df.to_dict('records')
        except Exception as e:
            return str(e)

    def getGroups(self):
        return self.groups

    def getAll(self):
        saida = {
            "user_data": self.getDataUser(),
            "tasks": self.getTasks(),
            "classes": self.getClasses(),
            "classes_with_group" : self.getClassesWithGroup(),
            #"last_classes": self.getLastClasses(),
            "notices": self.getNotices(),
            #"groups" : self.getGroups(),
        }
        return saida

    def getLastClasses(self):
        self.browser.open("https://sigaa.unifei.edu.br"+"/sigaa/portais/discente/turmas.jsf")
        disciplinas = self.browser.response().read()
        soup = bs(disciplinas, 'html.parser')

        tabela = soup.find(name="table", class_="listagem")
        periodos = [p.text for p in soup.find_all( class_="periodo")]

        df = pd.read_html(str(tabela),header=1)[0].iloc[:,0:4]

        aux = 0
        saida = dict()
        for periodo in periodos:
            if(aux == 0):
                saida[periodo] = df.iloc[1:(df.loc[df['Turma'] == float(periodos[1])].index.tolist()[0]),:].to_dict('records')
            else:
                if(aux+1==len(periodos)):
                    saida[periodo] = df.iloc[(df.loc[df['Turma'] == float(periodo)].index.tolist()[0]+1):-1,:].to_dict('records')
                else:
                    saida[periodo] = df.iloc[(df.loc[df['Turma'] == float(periodos[aux-1])].index.tolist()[0]+1):(df.loc[df['Turma'] == float(periodo)].index.tolist()[0]),:].to_dict('records')
            aux += 1

        return saida

    def getClassesWithGroup(self):
        self.browser.open("https://sigaa.unifei.edu.br"+"/sigaa/portais/discente/turmas.jsf")
        disciplinas = self.browser.response().read()
        soup = bs(disciplinas, 'html.parser')

        tabela = soup.find(name="table", class_="listagem")
        periodos = [p.text for p in soup.find_all( class_="periodo")]

        df = pd.read_html(str(tabela),header=1)[0].iloc[:,0:4]

        aux = 0
        saida = dict()
        for periodo in periodos:
            if(aux == 0):
                saida[periodo] = df.iloc[1:(df.loc[df['Turma'] == float(periodos[1])].index.tolist()[0]),:].to_dict('records')
            else:
                if(aux+1==len(periodos)):
                    saida[periodo] = df.iloc[(df.loc[df['Turma'] == float(periodo)].index.tolist()[0]+1):-1,:].to_dict('records')
                else:
                    saida[periodo] = df.iloc[(df.loc[df['Turma'] == float(periodos[aux-1])].index.tolist()[0]+1):(df.loc[df['Turma'] == float(periodo)].index.tolist()[0]),:].to_dict('records')
            aux += 1

        atual = saida[list(saida.keys())[0]]

        ctt = 0
        for disciplina in atual:
            atual[ctt]["Grupo"] = []
            for row in self.groups:
                if(row["codigo"] == disciplina["Disciplina"].split("-")[0].strip()):
                    atual[ctt]["Grupo"].append(row)
            ctt += 1

        return saida[list(saida.keys())[0]]


import einops
import torch
from kappadata import color_histogram
from torch.nn.functional import mse_loss

def color_histogram_regression_loss(preds, images, bins, temperature=1., loss_fn=mse_loss, reduction="mean"):
    channels = images.shape[1]
    if preds.ndim != 3:
        preds = einops.rearrange(preds, "bs (bins channels) -> bs channels bins", channels=channels, bins=bins)
    if temperature is None:
        # preds should be a distribution -> check if sums to 1
        sums = preds.sum(dim=2)
        assert torch.allclose(sums, torch.ones_like(sums)), sums
    else:
        # make preds into a distribution
        preds = (preds / temperature).softmax(dim=2)
    densities = color_histogram(images, bins=bins, density=True, batch_size=128)
    return loss_fn(preds, densities, reduction=reduction)


#%%
import cv2
import os
from os import path

def load_model():
	# model directory path predefined
	model_path = os.path.dirname(__file__)+"/model"
	for files in os.listdir(model_path):
		suffix = files.split(".")[-1]
		if suffix == "cfg":
			config = os.path.join(model_path,files)
		if suffix == "weights":
			weights = os.path.join(model_path,files)
	# load model
	net = cv2.dnn.readNetFromDarknet(config, weights)
	# determine only the *output* layer names that we need from YOLO
	temp = net.getLayerNames()
	layers = []
	for i in net.getUnconnectedOutLayers():
		# index = i[0] - 1
		layers.append(temp[i[0]-1])
	return net,layers


# -*- coding: utf-8 -*-
"""
Main entry point for the hubble daemon
"""

import argparse
import copy
import json
import logging
import math
import traceback
import os
import pprint
import re
import random
import signal
import socket
import sys
import time
import uuid
from datetime import datetime

import hubblestack.fileserver
import hubblestack.fileserver.gitfs
import hubblestack.modules.cmdmod
import hubblestack.utils
import hubblestack.utils.platform
import hubblestack.utils.jid
import hubblestack.utils.gitfs
import hubblestack.utils.path
from croniter import croniter

import hubblestack.loader
import hubblestack.utils.signing
import hubblestack.filter
import hubblestack.filter.filter_chain
import hubblestack.log
import hubblestack.log.splunk
import hubblestack.hec.opt
import hubblestack.utils.stdrec
from hubblestack import __version__
from hubblestack.hangtime import hangtime_wrapper
import hubblestack.status
import hubblestack.fileclient
import hubblestack.saltoverrides
import hubblestack.module_runner.runner
import hubblestack.module_runner.audit_runner
import hubblestack.module_runner.fdg_runner

log = logging.getLogger(__name__)
HSS = hubblestack.status.HubbleStatus(__name__, "schedule", "refresh_grains")

# Importing syslog fails on windows
if not hubblestack.utils.platform.is_windows():
    import syslog

__opts__ = {}
# This should work fine until we go to multiprocessing
SESSION_UUID = str(uuid.uuid4())

def run():
    """
    Set up program, daemonize if needed
    """
    try:
        load_config()
    except Exception as exc:
        print("An Error occurred while loading the config: %s", exc)
        raise
    # Create cache directory if not present
    if not os.path.isdir(__opts__["cachedir"]):
        os.makedirs(__opts__["cachedir"])
    try:
        main()
    except KeyboardInterrupt:
        pass

    clean_up_process(None, None)

def _clear_gitfs_locks():
    """Clear old locks and log the changes"""
    # Clear old locks
    if (
        "gitfs" in __opts__["fileserver_backend"]
        or "git" in __opts__["fileserver_backend"]
    ):
        git_objects = [
            hubblestack.utils.gitfs.GitFS(
                __opts__,
                __opts__["gitfs_remotes"],
                per_remote_overrides=hubblestack.fileserver.gitfs.PER_REMOTE_OVERRIDES,
                per_remote_only=hubblestack.fileserver.gitfs.PER_REMOTE_ONLY,
            )
        ]
        ret = {}
        for obj in git_objects:
            lock_type = "update"
            cleared, errors = hubblestack.fileserver.clear_lock(
                obj.clear_lock, "gitfs", remote=None, lock_type=lock_type
            )
            if cleared:
                ret.setdefault("cleared", []).extend(cleared)
            if errors:
                ret.setdefault("errors", []).extend(errors)
        if ret:
            log.info("One or more gitfs locks were removed: %s", ret)

def _emit_and_refresh_grains():
    """When the grains refresh frequency has expired, refresh grains and emit to syslog"""
    log.info("Refreshing grains")
    refresh_grains()
    last_grains_refresh = time.time()
    # Emit syslog at grains refresh frequency
    if not (hubblestack.utils.platform.is_windows()) and __opts__.get(
        "emit_grains_to_syslog", True
    ):
        default_grains_to_emit = [
            "system_uuid",
            "hubble_uuid",
            "session_uuid",
            "machine_id",
            "splunkindex",
            "cloud_details",
            "hubble_version",
            "localhost",
            "fqdn",
        ]
        grains_to_emit = []
        grains_to_emit.extend(
            __opts__.get("emit_grains_to_syslog_list", default_grains_to_emit)
        )
        emit_to_syslog(grains_to_emit)
    return last_grains_refresh

def _update_fileserver(file_client):
    """Update the filserver and the last_fc_update time"""
    try:
        file_client.channel.fs.update()
        last_fc_update = time.time()
    except Exception:
        retry = __opts__.get("fileserver_retry_rate", 900)
        last_fc_update += retry
        log.exception(
            "Exception thrown trying to update fileclient. "
            "Trying again in %s seconds.",
            retry,
        )
    return last_fc_update

def main():
    """
    Run the main hubble loop
    """
    # Initial fileclient setup
    _clear_gitfs_locks()
    # Setup fileclient
    log.info("Setting up the fileclient/fileserver")
    retry_count = __opts__.get("fileserver_retry_count_on_startup", None)
    retry_time = __opts__.get("fileserver_retry_rate_on_startup", 30)
    count = 0
    while True:
        try:
            file_client = hubblestack.fileclient.get_file_client(__opts__)
            file_client.channel.fs.update()
            last_fc_update = time.time()
            break
        except Exception:
            if (retry_count is None or count < retry_count) and not __opts__[
                "function"
            ]:
                log.exception(
                    "Exception thrown trying to setup fileclient. "
                    "Trying again in %s seconds.",
                    retry_time,
                )
                count += 1
                time.sleep(retry_time)
                continue
            else:
                log.exception("Exception thrown trying to setup fileclient. Exiting.")
                sys.exit(1)
    # Check for single function run
    if __opts__["function"]:
        run_function()
        sys.exit(0)
    last_grains_refresh = time.time() - __opts__["grains_refresh_frequency"]
    log.info("Starting main loop")
    pidfile_count = 0
    # pidfile_refresh in seconds, our scheduler deals in half-seconds
    pidfile_refresh = int(__opts__.get("pidfile_refresh", 60)) * 2
    while True:
        # Check if fileserver needs update
        if time.time() - last_fc_update >= __opts__["fileserver_update_frequency"]:
            last_fc_update = _update_fileserver(file_client)
        pidfile_count += 1
        if __opts__["daemonize"] and pidfile_count > pidfile_refresh:
            pidfile_count = 0
            create_pidfile()
        if time.time() - last_grains_refresh >= __opts__["grains_refresh_frequency"]:
            last_grains_refresh = _emit_and_refresh_grains()
        try:
            log.debug("Executing schedule")
            sf_count = schedule()
        except Exception as exc:
            log.exception("Error executing schedule: %s", exc)
            if isinstance(exc, KeyboardInterrupt):
                raise exc
        time.sleep(__opts__.get("scheduler_sleep_frequency", 0.5))

def getsecondsbycronexpression(base, cron_exp):
    """
    this function will return the seconds according to the cron
    expression provided in the hubble config
    """
    cron_iter = croniter(cron_exp, base)
    next_datetime = cron_iter.get_next(datetime)
    epoch_base_datetime = time.mktime(base.timetuple())
    epoch_datetime = time.mktime(next_datetime.timetuple())
    seconds = int(epoch_datetime) - int(epoch_base_datetime)
    return seconds

def getlastrunbycron(base, seconds):
    """
    this function will use the cron_exp provided in the hubble config to
    execute the hubble processes as per the scheduled cron time
    """
    epoch_base_datetime = time.mktime(base.timetuple())
    epoch_datetime = epoch_base_datetime
    current_time = time.time()
    while (epoch_datetime + seconds) < current_time:
        epoch_datetime = epoch_datetime + seconds
    last_run = epoch_datetime
    return last_run

def getlastrunbybuckets(buckets, seconds):
    """
    this function will use the host's ip to place the host in a bucket
    where each bucket executes hubble processes at a different time
    """
    buckets = int(buckets) if int(buckets) != 0 else 256
    host_ip = socket.gethostbyname(socket.gethostname())
    ips = host_ip.split(".")
    bucket_sum = (
        (int(ips[0]) * 256 * 256 * 256)
        + (int(ips[1]) * 256 * 256)
        + (int(ips[2]) * 256)
        + int(ips[3])
    )
    bucket = bucket_sum % buckets
    log.debug("bucket number is %d out of %d", bucket, buckets)
    current_time = time.time()
    base_time = seconds * (math.floor(current_time / seconds))
    splay = seconds / buckets
    seconds_between_buckets = splay
    random_int = random.randint(0, splay - 1) if splay != 0 else 0
    bucket_execution_time = base_time + (seconds_between_buckets * bucket) + random_int
    if bucket_execution_time < current_time:
        last_run = bucket_execution_time
    else:
        last_run = bucket_execution_time - seconds
    return last_run

@HSS.watch
def schedule():
    """
    Rudimentary single-pass scheduler

    If we find we miss some of the salt scheduler features we could potentially
    pull in some of that code.

    Schedule data should be placed in the config with the following format:

    .. code-block:: yaml

        schedule:
          job1:
            function: hubble.audit
            seconds: 3600
            splay: 100
            min_splay: 50
            args:
              - cis.centos-7-level-1-scored-v2-1-0
            kwargs:
              verbose: True
              show_profile: True
            returner: splunk_nova_return
            run_on_start: True

    Note that ``args``, ``kwargs``,``min_splay`` and ``splay`` are all optional. However, a
    scheduled job must always have a ``function`` and a time in ``seconds`` of
    how often to run the job.

    function
        Function to run in the format ``<module>.<function>``. Technically any
        salt module can be run in this way, but we recommend sticking to hubble
        functions. For simplicity, functions are run in the main daemon thread,
        so overloading the scheduler can result in functions not being run in
        a timely manner.

    seconds
        Frequency with which the job should be run, in seconds

    splay
        Randomized splay for the job, in seconds. A random number between <min_splay> and
        <splay> will be chosen and added to the ``seconds`` argument, to decide
        the true frequency. The splay will be chosen on first run, and will
        only change when the daemon is restarted. Optional.

    min_splay
        This parameters works in conjunction with <splay>. If a <min_splay> is provided, and random
        between <min_splay> and <splay> is chosen. If <min_splay> is not provided, it
        defaults to zero. Optional.

    args
        List of arguments for the function. Optional.

    kwargs
        Dict of keyword arguments for the function. Optional.

    returner
        String with a single returner, or list of returners to which we should
        send the results. Optional.

    run_on_start
        Whether to run the scheduled job on daemon start. Defaults to False. Optional.
    """
    sf_count = 0
    base = datetime(2018, 1, 1, 0, 0)
    schedule_config = __opts__.get("schedule", {})
    if "user_schedule" in __opts__ and isinstance(__opts__["user_schedule"], dict):
        schedule_config.update(__opts__["user_schedule"])
    for jobname, jobdata in schedule_config.items():
        try:
            # Error handling galore
            if not jobdata or not isinstance(jobdata, dict):
                log.error("Scheduled job %s does not have valid data", jobname)
                continue
            if "function" not in jobdata or "seconds" not in jobdata:
                log.error(
                    "Scheduled job %s is missing a ``function`` or ``seconds`` argument",
                    jobname,
                )
                continue
            func = jobdata["function"]
            if func not in __mods__:
                log.error(
                    "Scheduled job %s has a function %s which could not be found.",
                    jobname,
                    func,
                )
                continue
            try:
                if "cron" in jobdata:
                    seconds = getsecondsbycronexpression(base, jobdata["cron"])
                else:
                    seconds = int(jobdata["seconds"])
                splay = int(jobdata.get("splay", 0))
                min_splay = int(jobdata.get("min_splay", 0))
            except ValueError:
                log.error(
                    "Scheduled job %s has an invalid value for seconds or splay.",
                    jobname,
                )
            args = jobdata.get("args", [])
            if not isinstance(args, list):
                log.error(
                    "Scheduled job %s has args not formed as a list: %s", jobname, args
                )
            kwargs = jobdata.get("kwargs", {})
            if not isinstance(kwargs, dict):
                log.error(
                    "Scheduled job %s has kwargs not formed as a dict: %s",
                    jobname,
                    kwargs,
                )
            returners = jobdata.get("returner", [])
            if not isinstance(returners, list):
                returners = [returners]
            # Actually process the job
            run = _process_job(jobdata, splay, seconds, min_splay, base)
            if run:
                _execute_function(jobdata, func, returners, args, kwargs)
                sf_count += 1
        except:
            log.error(
                "Exception in running job: %s; continuing with next job...",
                jobname,
                exc_info=True,
            )
    return sf_count

def _execute_function(jobdata, func, returners, args, kwargs):
    """Run the scheduled function"""
    log.debug("Executing scheduled function %s", func)
    jobdata["last_run"] = time.time()

    # Actually run the function
    ret = __mods__[func](*args, **kwargs)

    if __opts__["log_level"] == "debug":
        log.debug("Job returned:\n%s", ret)
    for returner in returners:
        returner = "{0}.returner".format(returner)
        if returner not in __returners__:
            log.error("Could not find %s returner.", returner)
            continue
        log.debug("Returning job data to %s", returner)
        returner_ret = {
            "id": __grains__["id"],
            "jid": hubblestack.utils.jid.gen_jid(__opts__),
            "fun": func,
            "fun_args": args + ([kwargs] if kwargs else []),
            "return": ret,
        }
        __returners__[returner](returner_ret)

def _process_job(jobdata, splay, seconds, min_splay, base):
    run = False
    if "last_run" not in jobdata:
        if jobdata.get("run_on_start", False):
            if splay:
                # Run `splay` seconds in the future, by telling the scheduler we last ran it
                # `seconds - splay` seconds ago.
                jobdata["last_run"] = time.time() - (
                    seconds - random.randint(min_splay, splay)
                )
            else:
                # Run now
                run = True
                jobdata["last_run"] = time.time()
        else:
            if splay:
                # Run `seconds + splay` seconds in the future by telling the scheduler we last
                # ran it at now + `splay` seconds.
                jobdata["last_run"] = time.time() + random.randint(min_splay, splay)
            elif "buckets" in jobdata:
                # Place the host in a bucket and fix the execution time.
                jobdata["last_run"] = getlastrunbybuckets(jobdata["buckets"], seconds)
                log.debug("last_run according to bucket is %s", jobdata["last_run"])
            elif "cron" in jobdata:
                # execute the hubble process based on cron expression
                jobdata["last_run"] = getlastrunbycron(base, seconds)
            else:
                # Run in `seconds` seconds.
                jobdata["last_run"] = time.time()
    if jobdata["last_run"] < time.time() - seconds:
        run = True

    return run

def run_function():
    """
    Run a single function requested by the user
    """
    # Parse the args
    args = []
    kwargs = {}
    for arg in __opts__["args"]:
        if "=" in arg:
            kwarg, _, value = arg.partition("=")
            kwargs[kwarg] = value
        else:
            args.append(arg)
    log.debug("Parsed args: %s | Parsed kwargs: %s", args, kwargs)
    log.info("Executing user-requested function %s", __opts__["function"])

    mod_fun = __mods__.get(__opts__["function"])
    if not mod_fun or not callable(mod_fun):
        log.error("Function %s is not available, or not valid.", __opts__["function"])
        sys.exit(1)
    ret = mod_fun(*args, **kwargs)
    if __opts__["return"]:
        returner = "{0}.returner".format(__opts__["return"])
        if returner not in __returners__:
            log.error("Could not find %s returner.", returner)
        else:
            log.info("Returning job data to %s", returner)
            returner_ret = {
                "id": __grains__["id"],
                "jid": hubblestack.utils.jid.gen_jid(__opts__),
                "fun": __opts__["function"],
                "fun_args": args + ([kwargs] if kwargs else []),
                "return": ret,
            }
            __returners__[returner](returner_ret)
    # TODO instantiate the salt outputter system?
    if __opts__["json_print"]:
        print(json.dumps(ret))
    else:
        if not __opts__["no_pprint"]:
            pprint.pprint(ret)
        else:
            print(ret)

def load_config(args=None):
    """
    Load the config from configfile and load into imported salt modules
    """

    global __opts__

    # Parse arguments
    parsed_args = parse_args(args=args)

    # NOTE: if configfile isn't specified and None is passed to hubblestack.config.get_config
    # it will default to a platform specific file (see get_config() and DEFAULT_OPTS in hs.config)
    __opts__ = hubblestack.config.get_config(parsed_args.get("configfile"))

    # Loading default included config options and updating them in the main __opts__
    default_include_config_options = hubblestack.config.include_config(
        __opts__.get("default_include"), __opts__.get("conf_file"), verbose=False
    )
    __opts__.update(default_include_config_options)

    # we seem to have mixed feelings about whether to use __opts__ or parsed_args and mixed feelings
    # about whether it's spelled 'configfile' or 'conf_file'; so we just make them all work
    __opts__["configfile"] = parsed_args["configfile"] = __opts__["conf_file"]

    __opts__.update(parsed_args)
    __opts__["install_dir"] = hubblestack.syspaths.INSTALL_DIR
    __opts__["extension_modules"] = os.path.join(
        hubblestack.syspaths.CACHE_DIR, "extmods"
    )

    if __opts__["version"]:
        print(__version__)
        clean_up_process(None, None)
        sys.exit(0)
    if __opts__["buildinfo"]:
        try:
            from hubblestack import __buildinfo__
        except ImportError:
            __buildinfo__ = "NOT SET"
        print(__buildinfo__)
        clean_up_process(None, None)
        sys.exit(0)
    scan_proc = __opts__.get("scan_proc", False)
    if __opts__["daemonize"]:
        # before becoming a daemon, check for other procs and possibly send
        # them a signal 15 (otherwise refuse to run)
        if not __opts__.get("ignore_running", False):
            check_pidfile(kill_other=True, scan_proc=scan_proc)
        hubblestack.utils.daemonize()
        create_pidfile()
    elif (
        not __opts__["function"]
        and not __opts__["version"]
        and not __opts__["buildinfo"]
    ):
        # check the pidfile and possibly refuse to run (assuming this isn't a single function call)
        if not __opts__.get("ignore_running", False):
            check_pidfile(kill_other=False, scan_proc=scan_proc)
    # Optional sleep to wait for network
    time.sleep(int(__opts__.get("startup_sleep", 0)))
    _setup_signaling()
    # setup dirs for grains/returner/module
    _setup_dirs()
    _disable_boto_modules()
    _setup_logging(parsed_args)
    _setup_cached_uuid()
    refresh_grains(initial=True)
    if __mods__["config.get"]("splunklogging", False):
        hubblestack.log.setup_splunk_logger()
        hubblestack.log.emit_to_splunk(__grains__, "INFO", "hubblestack.grains_report")
        __mods__["conf_publisher.publish"]()

    return __opts__  # this is also a global, but the return is handy in tests/unittests

def _setup_signaling():
    """
    Hook the signal handler clean_up_process to trigger when certain signals are received
    """
    signal.signal(signal.SIGTERM, clean_up_process)
    signal.signal(signal.SIGINT, clean_up_process)
    signal.signal(signal.SIGABRT, clean_up_process)
    signal.signal(signal.SIGFPE, clean_up_process)
    signal.signal(signal.SIGILL, clean_up_process)
    signal.signal(signal.SIGSEGV, clean_up_process)
    if not hubblestack.utils.platform.is_windows():
        signal.signal(signal.SIGHUP, clean_up_process)
        signal.signal(signal.SIGQUIT, clean_up_process)

def _disable_boto_modules():
    """Disable the unneeded boto modules because they cause issues with the loader"""
    # Disable all of salt's boto modules, they give nothing but trouble to the loader
    disable_modules = __opts__.get("disable_modules", [])
    disable_modules.extend(
        [
            "boto3_elasticache",
            "boto3_route53",
            "boto3_sns",
            "boto_apigateway",
            "boto_asg",
            "boto_cfn",
            "boto_cloudfront",
            "boto_cloudtrail",
            "boto_cloudwatch_event",
            "boto_cloudwatch",
            "boto_cognitoidentity",
            "boto_datapipeline",
            "boto_dynamodb",
            "boto_ec2",
            "boto_efs",
            "boto_elasticache",
            "boto_elasticsearch_domain",
            "boto_elb",
            "boto_elbv2",
            "boto_iam",
            "boto_iot",
            "boto_kinesis",
            "boto_kms",
            "boto_lambda",
            "boto_rds",
            "boto_route53",
            "boto_s3_bucket",
            "boto_s3",
            "boto_secgroup",
            "boto_sns",
            "boto_sqs",
            "boto_ssm",
            "boto_vpc",
        ]
    )
    __opts__["disable_modules"] = disable_modules

def _setup_cached_uuid():
    """Get the cached uuid and cached system uui path, read the files
    and remove the cached uuid"""

    # Check for a cloned system with existing hubble_uuid
    def _get_uuid_from_system():
        query = '"SELECT uuid AS system_uuid FROM osquery_info;" --header=false --csv'

        # Prefer our /opt/osquery/osqueryi if present
        osqueryipaths = ("/opt/osquery/osqueryi", "osqueryi", "/usr/bin/osqueryi")
        for path in osqueryipaths:
            if hubblestack.utils.path.which(path):
                live_uuid = hubblestack.modules.cmdmod.run_stdout(
                    "{0} {1}".format(path, query), output_loglevel="quiet"
                )
                live_uuid = str(live_uuid).upper()
                if len(live_uuid) == 36:
                    return live_uuid
                return None
        # If osquery isn't available, attempt to get uuid from /sys path (linux only)
        try:
            with open(
                "/sys/devices/virtual/dmi/id/product_uuid", "r"
            ) as product_uuid_file:
                file_uuid = product_uuid_file.read()
            file_uuid = str(file_uuid).upper()
            if len(file_uuid) == 36:
                return file_uuid
            return None
        except Exception:
            return None

    cached_uuid_path = os.path.join(
        os.path.dirname(__opts__["configfile"]), "hubble_cached_uuid"
    )
    cached_system_uuid_path = os.path.join(
        os.path.dirname(__opts__["configfile"]), "hubble_cached_system_uuid"
    )
    try:
        if os.path.isfile(cached_uuid_path) and os.path.isfile(cached_system_uuid_path):
            with open(cached_uuid_path, "r") as cached_uuid_file, open(
                cached_system_uuid_path, "r"
            ) as cached_system_uuid_file:
                cached_uuid = cached_uuid_file.read()
                cached_system_uuid = cached_system_uuid_file.read()
            if cached_uuid != cached_system_uuid:
                live_uuid = _get_uuid_from_system()
                if live_uuid != cached_system_uuid:
                    log.error(
                        "potentially cloned system detected: System_uuid grain "
                        "previously saved on disk doesn't match live system value.\n"
                        "Resettig cached hubble_uuid value."
                    )
                    os.remove(cached_uuid_path)

    except Exception:
        log.exception(
            "Problem opening cache files while checking for previously cloned system"
        )

def _setup_logging(parsed_args):
    """
    Get logging options and setup logging
    """
    # Convert -vvv to log level
    if __opts__["log_level"] is None:
        # Default to 'error'
        __opts__["log_level"] = "error"
        # Default to more verbose if we're daemonizing
        if __opts__["daemonize"]:
            __opts__["log_level"] = "info"
    # Handle the explicit -vvv settings
    if __opts__["verbose"]:
        if __opts__["verbose"] == 1:
            __opts__["log_level"] = "warning"
        elif __opts__["verbose"] == 2:
            __opts__["log_level"] = "info"
        elif __opts__["verbose"] >= 3:
            __opts__["log_level"] = "debug"
    # Console logging is probably the same, but can be different
    console_logging_opts = {
        "log_level": __opts__.get("console_log_level", __opts__["log_level"]),
        "log_format": __opts__.get(
            "console_log_format", "%(asctime)s [%(levelname)-5s] %(message)s"
        ),
        "date_format": __opts__.get("console_log_date_format", "%H:%M:%S"),
    }
    file_logging_opts = {
        "log_file": __opts__.get("log_file", "/var/log/hubble"),
        "log_level": __opts__["log_level"],
        "log_format": __opts__.get(
            "log_format",
            "%(asctime)s,%(msecs)03d [%(levelname)-5s]"
            " [%(name)s:%(lineno)d]  %(message)s",
        ),
        "date_format": __opts__.get("log_date_format", "%Y-%m-%d %H:%M:%S"),
        "max_bytes": __opts__.get("logfile_maxbytes", 100000000),
        "backup_count": __opts__.get("logfile_backups", 1),
    }

    # Setup logging
    hubblestack.log.setup_console_logger(**console_logging_opts)
    if not parsed_args["skip_file_logger"]:
        hubblestack.log.setup_file_logger(**file_logging_opts)
        with open(__opts__["log_file"], "a") as _logfile:
            pass  # ensure the file exists before we set perms on it
        os.chmod(__opts__["log_file"], 0o600)

    configfile = parsed_args.get("configfile")
    if configfile and os.path.isfile(configfile):
        os.chmod(configfile, 0o600)

def _setup_dirs():
    """
    Setup module/grain/returner dirs
    """

    this_dir = os.path.dirname(__file__)

    # we have to uber-override and make sure our files dir is in root
    # and that root file systems are enabled

    this_root_files = os.path.join(this_dir, "files")

    if "file_roots" not in __opts__:
        __opts__["file_roots"] = dict(base=list())

    elif "base" not in __opts__["file_roots"]:
        __opts__["file_roots"]["base"] = [this_root_files]

    else:
        __opts__["file_roots"]["base"] = [this_root_files] + [
            x for x in __opts__["file_roots"]["base"] if x != this_root_files
        ]

    if "roots" not in __opts__["fileserver_backend"]:
        __opts__["fileserver_backend"].append("roots")

# 600s is a long time to get stuck loading grains and *not* be doing things
# like nova/pulsar. The SIGALRM will get caught by hubblestack.loader.raw_mod as an
# error in a grain -- probably whichever is broken/hung.
#
# The grain will simply be missing, but the next refresh_grains will try to
# pick it up again.  If the hang is transient, the grain will populate
# normally.
#
# repeats=True meaning: restart the signal itimer after firing the timeout
# exception, which salt catches. In this way, we can catch multiple hangs with
# a single timer. Each timer restart is a new 600s timeout.
#
# tag='hubble:rg' will appear in the logs to differentiate this from other
# hangtime_wrapper timers (if any)
@hangtime_wrapper(timeout=600, repeats=True, tag="hubble:rg")
@HSS.watch
def refresh_grains(initial=False):
    """
    Refresh the grains, pillar, utils, modules, and returners
    """
    global __opts__
    global __grains__
    global __utils__
    global __mods__
    global __pillar__
    global __returners__
    global __context__

    # 'POP' is for tracking persistent opts protection
    if os.environ.get("NOISY_POP_DEBUG"):
        log.error("POP refreshing grains (id=%d)", id(__opts__))

    persist, old_grains = {}, {}
    if initial:
        if not os.environ.get("NO_PRESERVE_OPTS"):
            if os.environ.get("NOISY_POP_DEBUG"):
                log.error("POP setting __opts__ to preservable (id=%d)", id(__opts__))
            hubblestack.loader.set_preservable_opts(__opts__)
        elif os.environ.get("NOISY_POP_DEBUG"):
            log.error(
                "POP we are not attemting to protect __opts__ from lazyloader reloads"
            )
    else:
        old_grains = copy.deepcopy(__grains__)
        for grain in __opts__.get("grains_persist", []):
            if grain in __grains__:
                persist[grain] = __grains__[grain]
        # Hardcode these core grains as persisting
        persist = {
            grain: __grains__[grain]
            for grain in ["hubble_version", "buildinfo"]
            if grain in __grains__
        }

    if initial:
        __context__ = {}
    if "grains" in __opts__:
        __opts__.pop("grains")
    if "pillar" in __opts__:
        __opts__.pop("pillar")
    __grains__ = hubblestack.loader.grains(__opts__)
    __grains__.update(persist)
    __grains__["session_uuid"] = SESSION_UUID

    # This was a weird one. In older versions of hubble the version and
    # buildinfo were not persisted automatically which means that if you
    # installed a new version without restarting hubble, grains refresh could
    # cause that old daemon to report grains as if it were the new version.
    # Now if this hubble_marker_3 grain is present you know you can trust the
    # hubble_version and buildinfo.
    __grains__["hubble_marker_3"] = True

    old_grains.update(__grains__)
    __grains__ = old_grains

    # Check for default gateway and fall back if necessary
    if (
        __grains__.get("ip_gw", None) is False
        and "fallback_fileserver_backend" in __opts__
    ):
        log.info("No default gateway detected; using fallback_fileserver_backend.")
        __opts__["fileserver_backend"] = __opts__["fallback_fileserver_backend"]

    __opts__["hubble_uuid"] = __grains__.get("hubble_uuid", None)
    __opts__["system_uuid"] = __grains__.get("system_uuid", None)
    __pillar__ = {}
    __opts__["grains"] = __grains__
    __opts__["pillar"] = __pillar__
    __utils__ = hubblestack.loader.utils(__opts__)
    __mods__ = hubblestack.loader.modules(
        __opts__, utils=__utils__, context=__context__
    )
    __returners__ = hubblestack.loader.returners(__opts__, __mods__)

    # the only things that turn up in here (and that get preserved)
    # are pulsar.queue, pulsar.notifier and cp.fileclient_###########
    # log.debug('keys in __context__: {}'.format(list(__context__)))

    hubblestack.utils.stdrec.__grains__ = __grains__
    hubblestack.utils.stdrec.__opts__ = __opts__

    hubblestack.hec.opt.__grains__ = __grains__
    hubblestack.hec.opt.__mods__ = __mods__
    hubblestack.hec.opt.__opts__ = __opts__

    hubblestack.filter.filter_chain.__mods__ = __mods__
    hubblestack.filter.filter_chain.__opts__ = __opts__

    hubblestack.log.splunk.__grains__ = __grains__
    hubblestack.log.splunk.__mods__ = __mods__
    hubblestack.log.splunk.__opts__ = __opts__

    hubblestack.status.__opts__ = __opts__
    hubblestack.status.__mods__ = __mods__

    hubblestack.utils.signing.__opts__ = __opts__
    hubblestack.utils.signing.__mods__ = __mods__

    hubblestack.module_runner.runner.__mods__ = __mods__
    hubblestack.module_runner.runner.__grains__ = __grains__
    hubblestack.module_runner.runner.__opts__ = __opts__

    hubblestack.module_runner.audit_runner.__mods__ = __mods__
    hubblestack.module_runner.audit_runner.__grains__ = __grains__
    hubblestack.module_runner.audit_runner.__opts__ = __opts__

    hubblestack.module_runner.fdg_runner.__mods__ = __mods__
    hubblestack.module_runner.fdg_runner.__grains__ = __grains__
    hubblestack.module_runner.fdg_runner.__opts__ = __opts__
    hubblestack.module_runner.fdg_runner.__returners__ = __returners__

    hubblestack.utils.signing.__mods__ = __mods__

    HSS.start_sigusr1_signal_handler()
    hubblestack.log.refresh_handler_std_info()
    clear_selective_context()

    if not initial and __mods__["config.get"]("splunklogging", False):
        hubblestack.log.emit_to_splunk(__grains__, "INFO", "hubblestack.grains_report")

def emit_to_syslog(grains_to_emit):
    """
    Emit grains and their values to syslog
    """
    try:
        # Avoid a syslog line to be longer than 1024 characters
        # Build syslog message
        syslog_list = ["hubble_syslog_message:"]
        for grain in grains_to_emit:
            if grain in __grains__:
                if bool(__grains__[grain]) and isinstance(__grains__[grain], dict):
                    for key, value in __grains__[grain].items():
                        syslog_list.append("{0}={1}".format(key, value))
                else:
                    syslog_list.append("{0}={1}".format(grain, __grains__[grain]))
        syslog_message = " ".join(syslog_list)
        log.info("Emitting some grains to syslog")
        syslog.openlog(logoption=syslog.LOG_PID)
        syslog.syslog(syslog_message)
    except Exception as exc:
        log.exception("An exception occurred on emitting a message to syslog: %s", exc)

def clear_selective_context():
    """
    Clear keys from __context__ global dictionary of salt
    Some modules saves data in this dictionary for system command execution
    If these keys exist in this dictionary, they just return data from there
    """
    global __context__

    # Fixing bug: Package list is not refreshed
    # clear the package list so that pkg module can fetch it as fresh in next cycle
    __context__.pop("pkg.list_pkgs", None)

def parse_args(args=None):
    """
    Parse command line arguments
    """
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "-d",
        "--daemonize",
        action="store_true",
        help="Whether to daemonize and background the process",
    )
    parser.add_argument(
        "-c",
        "--configfile",
        default=None,
        help="Pass in an alternative configuration file. Default: /etc/hubble/hubble",
    )
    parser.add_argument(
        "-p",
        "--no-pprint",
        help="Turn off pprint for single-function output",
        action="store_true",
    )
    parser.add_argument(
        "--skip-file-logger",
        help="Prevent logger from writing to /var/log/hubble.log",
        action="store_true",
    )
    parser.add_argument(
        "-v",
        "--verbose",
        action="count",
        help=(
            "Verbosity level. Use -v or -vv or -vvv for "
            "varying levels of verbosity. Note that -vv "
            "will be used by default in daemon mode."
        ),
    )
    parser.add_argument(
        "-r",
        "--return",
        default=None,
        help="Pass in a returner for single-function runs",
    )
    parser.add_argument(
        "--version", action="store_true", help="Show version information"
    )
    parser.add_argument(
        "--buildinfo", action="store_true", help="Show build information"
    )
    parser.add_argument(
        "function",
        nargs="?",
        default=None,
        help="Optional argument for the single function to be run",
    )
    parser.add_argument(
        "args", nargs="*", help="Any arguments necessary for a single function run"
    )
    parser.add_argument(
        "-j",
        "--json-print",
        action="store_true",
        help="Optional argument to print the output of single run function in json format",
    )
    parser.add_argument(
        "--ignore_running",
        action="store_true",
        help="Ignore any running hubble processes. This disables the pidfile.",
    )
    return vars(parser.parse_args(args=args))

def check_pidfile(kill_other=False, scan_proc=True):
    """
    Check to see if there's already a pidfile. If so, check to see if the
    indicated process is alive and is Hubble.

    kill_other
        Default false, if set to true, attempt to kill detected running Hubble processes;
        otherwise exit with an error.

    """
    pidfile_path = __opts__["pidfile"]
    if os.path.isfile(pidfile_path):
        with open(pidfile_path, "r") as pidfile:
            xpid = pidfile.readline().strip()
            try:
                xpid = int(xpid)
            except (TypeError, ValueError):
                xpid = 0
                log.warn('unable to parse pid="%d" in pidfile=%s', xpid, pidfile_path)
            if xpid:
                log.warn("pidfile=%s exists and contains pid=%d", pidfile_path, xpid)
                kill_other_or_sys_exit(xpid, kill_other=kill_other)
    if scan_proc:
        scan_proc_for_hubbles(kill_other=kill_other)

def kill_other_or_sys_exit(
    xpid, hname=r"hubble", ksig=signal.SIGTERM, kill_other=True, no_pgrp=True
):
    """Attempt to locate other hubbles using a cmdline regular expression and kill them when found.
    If killing the other processes fails (or kill_other is False), sys.exit instead.

    params:
      hname      :- the regular expression pattern to use to locate hubble (default: hubble)
      ksig       :- the signal to use to kill the other processes (default: signal.SIGTERM=15)
      kill_other :- (default: True); when false, don't attempt to kill,
                    just locate and exit (if found)
      no_pgrp    :- Avoid killing processes in this pgrp (avoid suicide). When no_pgrp is True,
                    invoke os.getprgp() to populate the actual value.

    caveats:
        There are some detailed notes on the process scanning in the function as comments.

        The most important caveat is that the hname regular expressions must match expecting
        that /proc/$$/cmdline text is null separated, not space separated.

        The other main caveat is that we can't actually examine the /proc/$$/exe file (that's
        always just a python). We have to scan the invocation text the kernel stored at launch.
        That text is not immutable and should not (normally) be relied upon for any purpose
        -- and this method does rely on it.
    """

    if no_pgrp is True:
        no_pgrp = os.getpgrp()
    if isinstance(no_pgrp, int):
        no_pgrp = str(no_pgrp)
    if os.path.isdir("/proc/{pid}".format(pid=xpid)):
        # NOTE: we'd prefer to check readlink(/proc/[pid]/exe), but that won't do
        # any good the /opt/whatever/bin/hubble is normally a text file with a
        # shebang; which the kernel picks up and uses to execute the real binary
        # with the "bin" file as an argument; so we'll have to live with cmdline
        pfile = "/proc/{pid}/cmdline".format(pid=xpid)
        log.debug("searching %s for hubble procs matching %s", pfile, hname)
        with open(pfile, "r") as pidfile:
            # NOTE: cmdline is actually null separated, not space separated
            # that shouldn't matter much for most hname regular expressions, but one never knows.
            cmdline = pidfile.readline().replace("\x00", " ").strip()
        if re.search(hname, cmdline):
            if no_pgrp:
                pstatfile = "/proc/{pid}/stat".format(pid=xpid)
                with open(pstatfile, "r") as fh2:
                    # NOTE: man proc(5) § /proc/[pid]/stat
                    # (pid, comm, state, ppid, pgrp, session, tty_nr, tpgid, flags, ...)
                    pgrp = fh2.readline().split()[4]
                    if pgrp == no_pgrp:
                        log.debug(
                            "process (%s) exists and seems to be a hubble, "
                            "but matches our process group (%s), ignoring",
                            xpid,
                            pgrp,
                        )
                        return False
            if kill_other:
                log.warn(
                    "process seems to still be alive and seems to be hubble,"
                    " attempting to shutdown"
                )
                os.kill(int(xpid), ksig)
                time.sleep(1)
                if os.path.isdir("/proc/{pid}".format(pid=xpid)):
                    log.error(
                        "fatal error: failed to shutdown process (pid=%s) successfully",
                        xpid,
                    )
                    sys.exit(1)
                else:
                    return True
            else:
                log.error("refusing to run while another hubble instance is running")
                sys.exit(1)
    else:
        # pidfile present, but nothing at that pid. Did we receive a sigterm?
        log.warning(
            "Pidfile found on startup, but no process at that pid. Did hubble receive a SIGTERM?"
        )
    return False

def scan_proc_for_hubbles(
    _proc_path="/proc",
    hname=r"^/\S+python.*?/opt/.*?hubble",
    kill_other=True,
    ksig=signal.SIGTERM,
):
    """look for other hubble processes and kill them or sys.exit()"""
    no_pgrp = str(os.getpgrp())
    rpid = re.compile(r"\d+")
    if os.path.isdir("/proc"):
        for dirname, dirs, _files in os.walk("/proc"):
            if dirname == "/proc":
                for pid in [i for i in dirs if rpid.match(i)]:
                    kill_other_or_sys_exit(
                        pid,
                        hname=hname,
                        kill_other=kill_other,
                        ksig=ksig,
                        no_pgrp=no_pgrp,
                    )
                break

def create_pidfile():
    """
    Create a pidfile after daemonizing
    """
    if not __opts__.get("ignore_running", False):
        pid = os.getpid()
        with open(__opts__["pidfile"], "w") as pidfile:
            pidfile.write(str(pid))

def clean_up_process(received_signal, frame):
    """
    Log any signals received. If a SIGTERM or SIGINT is received, clean up
    pidfile and anything else that needs to be cleaned up.
    """
    if received_signal is None and frame is None:
        if not __opts__.get("ignore_running", False) and \
           __opts__["daemonize"] and \
           os.path.isfile(__opts__["pidfile"]):
              os.remove(__opts__["pidfile"])
        sys.exit(0)
    try:
        if __mods__["config.get"]("splunklogging", False):
            hubblestack.log.emit_to_splunk(
                "Signal {0} detected".format(received_signal),
                "INFO",
                "hubblestack.signals",
            )
    finally:
        if received_signal == signal.SIGINT or received_signal == signal.SIGTERM:
            if not __opts__.get("ignore_running", False):
                if __opts__["daemonize"]:
                    if os.path.isfile(__opts__["pidfile"]):
                        os.remove(__opts__["pidfile"])
            sys.exit(0)


import requests
from django.conf import settings
from django.core.files.base import ContentFile

from auth.models import CustomUser as User
from thirdpartyaccounts.models import FacebookAccount, TwitterAccount

TWITTER_CONSUMER_KEY = getattr(settings, 'TWITTER_CONSUMER_KEY', '')
TWITTER_CONSUMER_SECRET = getattr(settings, 'TWITTER_CONSUMER_SECRET', '')
FACEBOOK_API_KEY = getattr(settings, 'FACEBOOK_API_KEY', '')
FACEBOOK_API_SECRET = getattr(settings, 'FACEBOOK_API_SECRET', '')

class TwitterAuthBackend(object):
    def authenticate(self, username):
        try:
            tpa = TwitterAccount.objects.get(username=username)
            user = User.objects.get(pk=tpa.user_id)
        except (TwitterAccount.DoesNotExist, User.DoesNotExist):
            return None
        return user

    def get_user(self, user_id):
        try:
            return User.objects.get(pk=user_id)
        except:
            return None

class FacebookAuthBackend(object):
    def authenticate(self, facebook=True, user=None):
        return user

    def get_user(self, user_id):
        try:
            return User.objects.get(pk=user_id)
        except:
            return None


from lib.equation import Equation

class FixedPoint(Equation):
    def solve_fixed(self, x, error_limit):
        error = 100
        n = 0
        while True:
            if error <= error_limit:
                break
            next_value = float(self.solve(x))
            error = self.get_percentual_error(x, next_value)
            self.sections.append({'x': str(next_value), 'error': str(error)})
            x = next_value

# if __name__ == '__main__':
#     fpoint = FixedPoint("E**-x", 'x')
#     fpoint.solve_fixed(1, 0.01)
#     print(fpoint.sections)


#!/usr/bin/env python
# Reads data from current sensor and publishes the data
# Written by Ashish Derhgawen (ashish.derhgawen@gmail.com)
# October, 2015

import rospy
import serial
from std_msgs.msg import Int16

ser = serial.Serial('/dev/ttyACM1', 9600)

def publisher():
    pub = rospy.Publisher('power', Int16, queue_size=10)
    rospy.init_node('power', anonymous=True)
    rate = rospy.Rate(100) # 100 Hz
    while not rospy.is_shutdown():
        sensor_data = ser.readline().rstrip('\r\n')

        if ',' in sensor_data: # Check if data is valid
            timestamp = sensor_data.split(",")[0]
            sensor_val = int(sensor_data.split(",")[1])

            rospy.loginfo(sensor_val)
            pub.publish(sensor_val)

        rate.sleep()

if __name__ == '__main__':
    try:
        publisher()
    except rospy.ROSInterruptException:
        pass


from torch.utils.data import Dataset, DataLoader
import numpy as np
import os
import torch
from PIL import Image
from torchvision import transforms

class ProjectDataset(Dataset):

    def __init__(self, base_dir):
        self.root_dir = base_dir
        self.episodes = [p for p in os.listdir(
            base_dir) if p.startswith('episode')]
        self.num_episodes = len(self.episodes)
        episode_len = []
        for episode in self.episodes:
            episode_dir = os.path.join(base_dir, episode)
            episode_len.append(len(os.listdir(episode_dir)) // 4)

        self.episode_len = episode_len

        self.transform = transforms.Compose([transforms.CenterCrop(512),
                                             transforms.ToTensor()])

    def __len__(self):
        return sum(self.episode_len)

    def __getitem__(self, idx):
        episode_num = idx % self.num_episodes
        episode = self.episodes[episode_num]
        point_ID = idx % self.episode_len[episode_num]
        img = self.transform(self.get_raw_img(episode, point_ID))
        seg_GT = self.transform(
            self.get_segmentation_img(episode, point_ID))[0, :, :]*255
        depth_img = self.transform(self.get_depth_img(episode, point_ID))
        depth_GT = torch.Tensor(self.depth_to_array(depth_img))

        return img, seg_GT, depth_GT

    def depth_to_array(self, image):
        img = np.array(image.permute((1, 2, 0)))
        # Apply (R + G * 256 + B * 256 * 256) / (256 * 256 * 256 - 1).
        normalized_depth = np.dot(img, [1.0, 256.0, 65536.0])
        normalized_depth /= 16777215.0  # (256.0 * 256.0 * 256.0 - 1.0)
        normalized_depth *= 1000 * 255
        return normalized_depth

    def get_raw_img(self, episode, point_ID):
        file_name = 'CentralRGB_' + str(point_ID).zfill(5) + '.png'
        file_path = os.path.join(
            self.root_dir, os.path.join(episode, file_name))
        return Image.open(file_path)

    def get_segmentation_img(self, episode, point_ID):
        file_name = 'CentralSemanticSeg_' + str(point_ID).zfill(5) + '.png'
        file_path = os.path.join(
            self.root_dir, os.path.join(episode, file_name))
        return Image.open(file_path)

    def get_depth_img(self, episode, point_ID):
        file_name = 'CentralDepth_' + str(point_ID).zfill(5) + '.png'
        file_path = os.path.join(
            self.root_dir, os.path.join(episode, file_name))
        return Image.open(file_path)

if __name__ == "__main__":
    dataset = ProjectDataset(base_dir='episodes/')

    dataloader = DataLoader(
        dataset, batch_size=1, shuffle=True, num_workers=1)
    # dataloader = DataLoader(vqa_dataset, batch_size=100)
    print(dataloader)
    # sample = vqa_dataset[1]
    # print(sample)

    for i_batch, sample_batched in enumerate(dataloader):
        print(i_batch, torch.max(
            sample_batched[1]), torch.min(sample_batched[1]))

        img, seg_GT, depth_GT = sample_batched
        print(img.shape)
        print(depth_GT.min())
        print(depth_GT.max())
        print(seg_GT.shape)
        print(seg_GT.max())
        print(seg_GT.min())
        print(depth_GT.shape)
        if(i_batch == 10):
            break
    # print(i, sample['image'].shape, sample['landmarks'].shape)


import collections
import numpy
import dataset
import image_utils
from random import shuffle
from tensorflow.python.framework import dtypes

Datasets = collections.namedtuple('Datasets', ['train', 'validation', 'test'])

def read_data_semeion(fname = 'dataset/semeion/semeion.data'):
	file = open(fname, 'r')
	lines = file.readlines()

	width = 16
	height = 16
	size = width * height
	classes = 10

	images = [];
	labels = [];
	fnumber = 0;

	for line in lines:
		data = line.split(' ')
		image = [];
		label = [];

		for i in range(0, size):
			image.append(int(float(data[i])))
		images.append(image)

		for i in range(size, size + classes):
			label.append(int(float(data[i])))
		labels.append(label)

		fnumber += 1
		#if fnumber < 10:
			#image_utils.show(image, width, height)
		#image_utils.save('./dataset/semeion/images/' + str(fnumber) + '.png', array, width, height)

	#Shuffle data
	images_shuffle = []
	labels_shuffle = []
	indexes = list(range(len(images)))
	shuffle(indexes)
	for i in indexes:
		images_shuffle.append(images[i])
		labels_shuffle.append(labels[i])

	images = images_shuffle
	labels = labels_shuffle

	samples = len(lines)
	train_samples = 1300
	test_samples = 1100

	#Train set
	image_train = numpy.array(images[:train_samples], dtype=numpy.uint8)
	image_train = image_train.reshape(train_samples, width, height, 1)

	label_train = numpy.array(labels[:train_samples], dtype=numpy.uint8)

	train = dataset.DataSet(image_train, label_train, reshape=True)

	#test set
	image_test = numpy.array(images[test_samples:], dtype=numpy.uint8)
	image_test = image_test.reshape(samples - test_samples, width, height, 1)

	label_test = numpy.array(labels[test_samples:], dtype=numpy.uint8)

	test = dataset.DataSet(image_test, label_test, reshape=True)

	return Datasets(train=train, validation=data, test=test)


import hashlib
import time
import sqlite3
import json

from merkletools import MerkleTools

from Transaction import Transaction

def calculate_hash(index, previous_hash, timestamp, data):
    return hashlib.sha256(
        (str(index) + str(previous_hash) + str(timestamp) + json.dumps(data)).encode('utf-8')).hexdigest()

class Block:

    def __init__(self, index, transaction_list, merkelTree, merkelTreeRoot, timestamp, previous_hash, proof=None):
        self.index = index
        self.timestamp = timestamp
        self.transaction_list = transaction_list
        self.merkelTree = merkelTree
        self.merkelTreeRoot = merkelTreeRoot
        self.previous_hash = previous_hash
        self.hash = calculate_hash(index, timestamp, merkelTreeRoot, previous_hash)
        self.proof = proof

    def calculate_hash(self):
        return hashlib.sha256(
            str(self.index) + str(self.previous_hash) + str(self.timestamp) + json.dumps(
                self.merkelTreeRoot)).hexdigest()

    def toJson(self):
        return json.dumps(
            {
                "index": self.index,
                "timestamp": self.timestamp,
                "merkelTreeRoot": self.merkelTreeRoot,
                "previous_hash": self.previous_hash,
                "hash": self.hash,
                "proof": self.proof,
            }
        )

    def to_json_no_proof(self):
        return json.dumps(
            {
                "index": self.index,
                "timestamp": self.timestamp,
                "merkelTreeRoot": self.merkelTreeRoot,
                "previous_hash": self.previous_hash,
                "hash": self.hash
            }
        )

def buildTree(transactions: list[Transaction]):
    mt = MerkleTools(hash_type="md5")
    for t in transactions:
        mt.add_leaf(t.toJson(), True)
        print(t)
    mt.make_tree()
    return mt.get_merkle_root(), mt

def validate_transactions(transactions: list[Transaction], mt: MerkleTools):
    hash_function = mt.hash_function
    for t in transactions:
        v = t.toJson().encode('utf-8')
        v = hash_function(v).hexdigest()
        v = bytearray.fromhex(v)
        if v not in mt.leaves:
            raise Exception("Invalid transaction in the list")


import email
from email.header import decode_header

def decode_mail_header(value, default_charset='us-ascii'):
	"""
	Decode a header value into a unicode string.
	"""
	try:
		headers=email.header.decode_header(value)
	except email.errors.HeaderParseError:
		return value.encode('us-ascii', 'replace').decode('us-ascii')
	else:
		for i, (text, charset) in enumerate(headers):
			try:
				headers[i]=text.decode(charset or 'us-ascii', 'replace')
			except LookupError:
				# if the charset is unknown, force default
				headers[i]=text.decode(default_charset, 'replace')

		return u"".join(headers)

def get_mail_addresses(message, header_name):
	"""
	retrieve all email addresses from one message header

	"""
	addresses = email.utils.getaddresses(h for h in message.get_all(header_name, []))

	for i, (address_name, address) in enumerate(addresses):
		addresses[i]={'name': decode_mail_header(address_name), 'email': address}

	return addresses