"""
Ported to Python 3.
"""

from future.utils import bchr

import struct, time, os, sys

from twisted.python import usage, failure
from twisted.internet import defer
from foolscap.logging import cli as foolscap_cli

from allmydata.scripts.common import BaseOptions
from allmydata import uri
from allmydata.storage.mutable import MutableShareFile
from allmydata.storage.immutable import ShareFile
from allmydata.mutable.layout import unpack_share
from allmydata.mutable.layout import MDMFSlotReadProxy
from allmydata.mutable.common import NeedMoreDataError
from allmydata.immutable.layout import ReadBucketProxy
from allmydata.util import base32
from allmydata.util.encodingutil import quote_output
from allmydata.scripts.types_ import SubCommands

class DumpOptions(BaseOptions):
    def getSynopsis(self):
        return "Usage: tahoe [global-options] debug dump-share SHARE_FILENAME"

    optFlags = [
        ["offsets", None, "Display a table of section offsets."],
        ["leases-only", None, "Dump leases but not CHK contents."],
        ]

    description = """
Print lots of information about the given share, by parsing the share's
contents. This includes share type, lease information, encoding parameters,
hash-tree roots, public keys, and segment sizes. This command also emits a
verify-cap for the file that uses the share.

 tahoe debug dump-share testgrid/node-3/storage/shares/4v/4vozh77tsrw7mdhnj7qvp5ky74/0
"""

    def parseArgs(self, filename):
        from allmydata.util.encodingutil import argv_to_abspath
        self['filename'] = argv_to_abspath(filename)

def dump_share(options):
    from allmydata.storage.mutable import MutableShareFile
    from allmydata.util.encodingutil import quote_output

    out = options.stdout

    # check the version, to see if we have a mutable or immutable share
    print("share filename: %s" % quote_output(options['filename']), file=out)

    with open(options['filename'], "rb") as f:
        if MutableShareFile.is_valid_header(f.read(32)):
            return dump_mutable_share(options)
        # otherwise assume it's immutable
        return dump_immutable_share(options)

def dump_immutable_share(options):
    from allmydata.storage.immutable import ShareFile

    out = options.stdout
    f = ShareFile(options['filename'])
    if not options["leases-only"]:
        dump_immutable_chk_share(f, out, options)
    dump_immutable_lease_info(f, out)
    print(file=out)
    return 0

def dump_immutable_chk_share(f, out, options):
    from allmydata import uri
    from allmydata.util import base32
    from allmydata.immutable.layout import ReadBucketProxy
    from allmydata.util.encodingutil import quote_output, to_bytes

    # use a ReadBucketProxy to parse the bucket and find the uri extension
    bp = ReadBucketProxy(None, None, '')
    offsets = bp._parse_offsets(f.read_share_data(0, 0x44))
    print("%20s: %d" % ("version", bp._version), file=out)
    seek = offsets['uri_extension']
    length = struct.unpack(bp._fieldstruct,
                           f.read_share_data(seek, bp._fieldsize))[0]
    seek += bp._fieldsize
    UEB_data = f.read_share_data(seek, length)

    unpacked = uri.unpack_extension_readable(UEB_data)
    keys1 = ("size", "num_segments", "segment_size",
             "needed_shares", "total_shares")
    keys2 = ("codec_name", "codec_params", "tail_codec_params")
    keys3 = ("plaintext_hash", "plaintext_root_hash",
             "crypttext_hash", "crypttext_root_hash",
             "share_root_hash", "UEB_hash")
    display_keys = {"size": "file_size"}

    def to_string(v):
        if isinstance(v, bytes):
            return str(v, "utf-8")
        else:
            return str(v)

    for k in keys1:
        if k in unpacked:
            dk = display_keys.get(k, k)
            print("%20s: %s" % (dk, to_string(unpacked[k])), file=out)
    print(file=out)
    for k in keys2:
        if k in unpacked:
            dk = display_keys.get(k, k)
            print("%20s: %s" % (dk, to_string(unpacked[k])), file=out)
    print(file=out)
    for k in keys3:
        if k in unpacked:
            dk = display_keys.get(k, k)
            print("%20s: %s" % (dk, to_string(unpacked[k])), file=out)

    leftover = set(unpacked.keys()) - set(keys1 + keys2 + keys3)
    if leftover:
        print(file=out)
        print("LEFTOVER:", file=out)
        for k in sorted(leftover):
            print("%20s: %s" % (k, to_string(unpacked[k])), file=out)

    # the storage index isn't stored in the share itself, so we depend upon
    # knowing the parent directory name to get it
    pieces = options['filename'].split(os.sep)
    if len(pieces) >= 2:
        piece = to_bytes(pieces[-2])
        if base32.could_be_base32_encoded(piece):
            storage_index = base32.a2b(piece)
            uri_extension_hash = base32.a2b(unpacked["UEB_hash"])
            u = uri.CHKFileVerifierURI(storage_index, uri_extension_hash,
                                      unpacked["needed_shares"],
                                      unpacked["total_shares"], unpacked["size"])
            verify_cap = u.to_string()
            print("%20s: %s" % ("verify-cap", quote_output(verify_cap, quotemarks=False)), file=out)

    sizes = {}
    sizes['data'] = (offsets['plaintext_hash_tree'] -
                           offsets['data'])
    sizes['validation'] = (offsets['uri_extension'] -
                           offsets['plaintext_hash_tree'])
    sizes['uri-extension'] = len(UEB_data)
    print(file=out)
    print(" Size of data within the share:", file=out)
    for k in sorted(sizes):
        print("%20s: %s" % (k, sizes[k]), file=out)

    if options['offsets']:
        print(file=out)
        print(" Section Offsets:", file=out)
        print("%20s: %s" % ("share data", f._data_offset), file=out)
        for k in ["data", "plaintext_hash_tree", "crypttext_hash_tree",
                  "block_hashes", "share_hashes", "uri_extension"]:
            name = {"data": "block data"}.get(k,k)
            offset = f._data_offset + offsets[k]
            print("  %20s: %s   (0x%x)" % (name, offset, offset), file=out)
        print("%20s: %s" % ("leases", f._lease_offset), file=out)

def dump_immutable_lease_info(f, out):
    # display lease information too
    print(file=out)
    leases = list(f.get_leases())
    if leases:
        for i,lease in enumerate(leases):
            when = format_expiration_time(lease.get_expiration_time())
            print(" Lease #%d: owner=%d, expire in %s" \
                  % (i, lease.owner_num, when), file=out)
    else:
        print(" No leases.", file=out)

def format_expiration_time(expiration_time):
    now = time.time()
    remains = expiration_time - now
    when = "%ds" % remains
    if remains > 24*3600:
        when += " (%d days)" % (remains // (24*3600))
    elif remains > 3600:
        when += " (%d hours)" % (remains // 3600)
    return when

def dump_mutable_share(options):
    from allmydata.storage.mutable import MutableShareFile
    from allmydata.util import base32, idlib
    out = options.stdout
    m = MutableShareFile(options['filename'])
    f = open(options['filename'], "rb")
    WE, nodeid = m._read_write_enabler_and_nodeid(f)
    num_extra_leases = m._read_num_extra_leases(f)
    data_length = m._read_data_length(f)
    extra_lease_offset = m._read_extra_lease_offset(f)
    container_size = extra_lease_offset - m.DATA_OFFSET
    leases = list(m._enumerate_leases(f))

    share_type = "unknown"
    f.seek(m.DATA_OFFSET)
    version = f.read(1)
    if version == b"\x00":
        # this slot contains an SMDF share
        share_type = "SDMF"
    elif version == b"\x01":
        share_type = "MDMF"
    f.close()

    print(file=out)
    print("Mutable slot found:", file=out)
    print(" share_type: %s" % share_type, file=out)
    print(" write_enabler: %s" % str(base32.b2a(WE), "utf-8"), file=out)
    print(" WE for nodeid: %s" % idlib.nodeid_b2a(nodeid), file=out)
    print(" num_extra_leases: %d" % num_extra_leases, file=out)
    print(" container_size: %d" % container_size, file=out)
    print(" data_length: %d" % data_length, file=out)
    if leases:
        for (leasenum, lease) in leases:
            print(file=out)
            print(" Lease #%d:" % leasenum, file=out)
            print("  ownerid: %d" % lease.owner_num, file=out)
            when = format_expiration_time(lease.get_expiration_time())
            print("  expires in %s" % when, file=out)
            print("  renew_secret: %s" % lease.present_renew_secret(), file=out)
            print("  cancel_secret: %s" % lease.present_cancel_secret(), file=out)
            print("  secrets are for nodeid: %s" % idlib.nodeid_b2a(lease.nodeid), file=out)
    else:
        print("No leases.", file=out)
    print(file=out)

    if share_type == "SDMF":
        dump_SDMF_share(m, data_length, options)
    elif share_type == "MDMF":
        dump_MDMF_share(m, data_length, options)

    return 0

def dump_SDMF_share(m, length, options):
    from allmydata.mutable.layout import unpack_share, unpack_header
    from allmydata.mutable.common import NeedMoreDataError
    from allmydata.util import base32, hashutil
    from allmydata.uri import SSKVerifierURI
    from allmydata.util.encodingutil import quote_output, to_bytes

    offset = m.DATA_OFFSET

    out = options.stdout

    f = open(options['filename'], "rb")
    f.seek(offset)
    data = f.read(min(length, 2000))
    f.close()

    try:
        pieces = unpack_share(data)
    except NeedMoreDataError as e:
        # retry once with the larger size
        size = e.needed_bytes
        f = open(options['filename'], "rb")
        f.seek(offset)
        data = f.read(min(length, size))
        f.close()
        pieces = unpack_share(data)

    (seqnum, root_hash, IV, k, N, segsize, datalen,
     pubkey, signature, share_hash_chain, block_hash_tree,
     share_data, enc_privkey) = pieces
    (ig_version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize,
     ig_datalen, offsets) = unpack_header(data)

    print(" SDMF contents:", file=out)
    print("  seqnum: %d" % seqnum, file=out)
    print("  root_hash: %s" % str(base32.b2a(root_hash), "utf-8"), file=out)
    print("  IV: %s" % str(base32.b2a(IV), "utf-8"), file=out)
    print("  required_shares: %d" % k, file=out)
    print("  total_shares: %d" % N, file=out)
    print("  segsize: %d" % segsize, file=out)
    print("  datalen: %d" % datalen, file=out)
    print("  enc_privkey: %d bytes" % len(enc_privkey), file=out)
    print("  pubkey: %d bytes" % len(pubkey), file=out)
    print("  signature: %d bytes" % len(signature), file=out)
    share_hash_ids = ",".join(sorted([str(hid)
                                      for hid in share_hash_chain.keys()]))
    print("  share_hash_chain: %s" % share_hash_ids, file=out)
    print("  block_hash_tree: %d nodes" % len(block_hash_tree), file=out)

    # the storage index isn't stored in the share itself, so we depend upon
    # knowing the parent directory name to get it
    pieces = options['filename'].split(os.sep)
    if len(pieces) >= 2:
        piece = to_bytes(pieces[-2])
        if base32.could_be_base32_encoded(piece):
            storage_index = base32.a2b(piece)
            fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey)
            u = SSKVerifierURI(storage_index, fingerprint)
            verify_cap = u.to_string()
            print("  verify-cap:", quote_output(verify_cap, quotemarks=False), file=out)

    if options['offsets']:
        # NOTE: this offset-calculation code is fragile, and needs to be
        # merged with MutableShareFile's internals.
        print(file=out)
        print(" Section Offsets:", file=out)
        def printoffset(name, value, shift=0):
            print("%s%20s: %s   (0x%x)" % (" "*shift, name, value, value), file=out)
        printoffset("first lease", m.HEADER_SIZE)
        printoffset("share data", m.DATA_OFFSET)
        o_seqnum = m.DATA_OFFSET + struct.calcsize(">B")
        printoffset("seqnum", o_seqnum, 2)
        o_root_hash = m.DATA_OFFSET + struct.calcsize(">BQ")
        printoffset("root_hash", o_root_hash, 2)
        for k in ["signature", "share_hash_chain", "block_hash_tree",
                  "share_data",
                  "enc_privkey", "EOF"]:
            name = {"share_data": "block data",
                    "EOF": "end of share data"}.get(k,k)
            offset = m.DATA_OFFSET + offsets[k]
            printoffset(name, offset, 2)
        f = open(options['filename'], "rb")
        printoffset("extra leases", m._read_extra_lease_offset(f) + 4)
        f.close()

    print(file=out)

def dump_MDMF_share(m, length, options):
    from allmydata.mutable.layout import MDMFSlotReadProxy
    from allmydata.util import base32, hashutil
    from allmydata.uri import MDMFVerifierURI
    from allmydata.util.encodingutil import quote_output, to_bytes

    offset = m.DATA_OFFSET
    out = options.stdout

    f = open(options['filename'], "rb")
    storage_index = None; shnum = 0

    class ShareDumper(MDMFSlotReadProxy):
        def _read(self, readvs, force_remote=False, queue=False):
            data = []
            for (where,length) in readvs:
                f.seek(offset+where)
                data.append(f.read(length))
            return defer.succeed({shnum: data})

    p = ShareDumper(None, storage_index, shnum)
    def extract(func):
        stash = []
        # these methods return Deferreds, but we happen to know that they run
        # synchronously when not actually talking to a remote server
        d = func()
        d.addCallback(stash.append)
        return stash[0]

    verinfo = extract(p.get_verinfo)
    encprivkey = extract(p.get_encprivkey)
    signature = extract(p.get_signature)
    pubkey = extract(p.get_verification_key)
    block_hash_tree = extract(p.get_blockhashes)
    share_hash_chain = extract(p.get_sharehashes)
    f.close()

    (seqnum, root_hash, salt_to_use, segsize, datalen, k, N, prefix,
     offsets) = verinfo

    print(" MDMF contents:", file=out)
    print("  seqnum: %d" % seqnum, file=out)
    print("  root_hash: %s" % str(base32.b2a(root_hash), "utf-8"), file=out)
    #print("  IV: %s" % base32.b2a(IV), file=out)
    print("  required_shares: %d" % k, file=out)
    print("  total_shares: %d" % N, file=out)
    print("  segsize: %d" % segsize, file=out)
    print("  datalen: %d" % datalen, file=out)
    print("  enc_privkey: %d bytes" % len(encprivkey), file=out)
    print("  pubkey: %d bytes" % len(pubkey), file=out)
    print("  signature: %d bytes" % len(signature), file=out)
    share_hash_ids = ",".join([str(hid)
                               for hid in sorted(share_hash_chain.keys())])
    print("  share_hash_chain: %s" % share_hash_ids, file=out)
    print("  block_hash_tree: %d nodes" % len(block_hash_tree), file=out)

    # the storage index isn't stored in the share itself, so we depend upon
    # knowing the parent directory name to get it
    pieces = options['filename'].split(os.sep)
    if len(pieces) >= 2:
        piece = to_bytes(pieces[-2])
        if base32.could_be_base32_encoded(piece):
            storage_index = base32.a2b(piece)
            fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey)
            u = MDMFVerifierURI(storage_index, fingerprint)
            verify_cap = u.to_string()
            print("  verify-cap:", quote_output(verify_cap, quotemarks=False), file=out)

    if options['offsets']:
        # NOTE: this offset-calculation code is fragile, and needs to be
        # merged with MutableShareFile's internals.

        print(file=out)
        print(" Section Offsets:", file=out)
        def printoffset(name, value, shift=0):
            print("%s%.20s: %s   (0x%x)" % (" "*shift, name, value, value), file=out)
        printoffset("first lease", m.HEADER_SIZE, 2)
        printoffset("share data", m.DATA_OFFSET, 2)
        o_seqnum = m.DATA_OFFSET + struct.calcsize(">B")
        printoffset("seqnum", o_seqnum, 4)
        o_root_hash = m.DATA_OFFSET + struct.calcsize(">BQ")
        printoffset("root_hash", o_root_hash, 4)
        for k in ["enc_privkey", "share_hash_chain", "signature",
                  "verification_key", "verification_key_end",
                  "share_data", "block_hash_tree", "EOF"]:
            name = {"share_data": "block data",
                    "verification_key": "pubkey",
                    "verification_key_end": "end of pubkey",
                    "EOF": "end of share data"}.get(k,k)
            offset = m.DATA_OFFSET + offsets[k]
            printoffset(name, offset, 4)
        f = open(options['filename'], "rb")
        printoffset("extra leases", m._read_extra_lease_offset(f) + 4, 2)
        f.close()

    print(file=out)

class DumpCapOptions(BaseOptions):
    def getSynopsis(self):
        return "Usage: tahoe [global-options] debug dump-cap [options] FILECAP"
    optParameters = [
        ["nodeid", "n",
         None, "Specify the storage server nodeid (ASCII), to construct WE and secrets."],
        ["client-secret", "c", None,
         "Specify the client's base secret (ASCII), to construct secrets."],
        ["client-dir", "d", None,
         "Specify the client's base directory, from which a -c secret will be read."],
        ]
    def parseArgs(self, cap):
        self.cap = cap

    description = """
Print information about the given cap-string (aka: URI, file-cap, dir-cap,
read-cap, write-cap). The URI string is parsed and unpacked. This prints the
type of the cap, its storage index, and any derived keys.

 tahoe debug dump-cap URI:SSK-Verifier:4vozh77tsrw7mdhnj7qvp5ky74:q7f3dwz76sjys4kqfdt3ocur2pay3a6rftnkqmi2uxu3vqsdsofq

This may be useful to determine if a read-cap and a write-cap refer to the
same time, or to extract the storage-index from a file-cap (to then use with
find-shares)

If additional information is provided (storage server nodeid and/or client
base secret), this command will compute the shared secrets used for the
write-enabler and for lease-renewal.
"""

def dump_cap(options):
    from allmydata import uri
    from allmydata.util import base32
    from base64 import b32decode
    from urllib.parse import unquote, urlparse

    out = options.stdout
    cap = options.cap
    nodeid = None
    if options['nodeid']:
        nodeid = b32decode(options['nodeid'].upper())
    secret = None
    if options['client-secret']:
        secret = base32.a2b(options['client-secret'].encode("ascii"))
    elif options['client-dir']:
        secretfile = os.path.join(options['client-dir'], "private", "secret")
        try:
            secret = base32.a2b(open(secretfile, "rb").read().strip())
        except EnvironmentError:
            pass

    if cap.startswith("http"):
        scheme, netloc, path, params, query, fragment = urlparse(cap)
        assert path.startswith("/uri/")
        cap = unquote(path[len("/uri/"):])

    u = uri.from_string(cap)

    print(file=out)
    dump_uri_instance(u, nodeid, secret, out)

def _dump_secrets(storage_index, secret, nodeid, out):
    from allmydata.util import hashutil
    from allmydata.util import base32

    if secret:
        crs = hashutil.my_renewal_secret_hash(secret)
        print(" client renewal secret:", str(base32.b2a(crs), "ascii"), file=out)
        frs = hashutil.file_renewal_secret_hash(crs, storage_index)
        print(" file renewal secret:", str(base32.b2a(frs), "ascii"), file=out)
        if nodeid:
            renew = hashutil.bucket_renewal_secret_hash(frs, nodeid)
            print(" lease renewal secret:", str(base32.b2a(renew), "ascii"), file=out)
        ccs = hashutil.my_cancel_secret_hash(secret)
        print(" client cancel secret:", str(base32.b2a(ccs), "ascii"), file=out)
        fcs = hashutil.file_cancel_secret_hash(ccs, storage_index)
        print(" file cancel secret:", str(base32.b2a(fcs), "ascii"), file=out)
        if nodeid:
            cancel = hashutil.bucket_cancel_secret_hash(fcs, nodeid)
            print(" lease cancel secret:", str(base32.b2a(cancel), "ascii"), file=out)

def dump_uri_instance(u, nodeid, secret, out, show_header=True):
    from allmydata import uri
    from allmydata.storage.server import si_b2a
    from allmydata.util import base32, hashutil
    from allmydata.util.encodingutil import quote_output

    if isinstance(u, uri.CHKFileURI):
        if show_header:
            print("CHK File:", file=out)
        print(" key:", str(base32.b2a(u.key), "ascii"), file=out)
        print(" UEB hash:", str(base32.b2a(u.uri_extension_hash), "ascii"), file=out)
        print(" size:", u.size, file=out)
        print(" k/N: %d/%d" % (u.needed_shares, u.total_shares), file=out)
        print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out)
        _dump_secrets(u.get_storage_index(), secret, nodeid, out)
    elif isinstance(u, uri.CHKFileVerifierURI):
        if show_header:
            print("CHK Verifier URI:", file=out)
        print(" UEB hash:", str(base32.b2a(u.uri_extension_hash), "ascii"), file=out)
        print(" size:", u.size, file=out)
        print(" k/N: %d/%d" % (u.needed_shares, u.total_shares), file=out)
        print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out)

    elif isinstance(u, uri.LiteralFileURI):
        if show_header:
            print("Literal File URI:", file=out)
        print(" data:", quote_output(u.data), file=out)

    elif isinstance(u, uri.WriteableSSKFileURI): # SDMF
        if show_header:
            print("SDMF Writeable URI:", file=out)
        print(" writekey:", str(base32.b2a(u.writekey), "ascii"), file=out)
        print(" readkey:", str(base32.b2a(u.readkey), "ascii"), file=out)
        print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out)
        print(" fingerprint:", str(base32.b2a(u.fingerprint), "ascii"), file=out)
        print(file=out)
        if nodeid:
            we = hashutil.ssk_write_enabler_hash(u.writekey, nodeid)
            print(" write_enabler:", str(base32.b2a(we), "ascii"), file=out)
            print(file=out)
        _dump_secrets(u.get_storage_index(), secret, nodeid, out)
    elif isinstance(u, uri.ReadonlySSKFileURI):
        if show_header:
            print("SDMF Read-only URI:", file=out)
        print(" readkey:", str(base32.b2a(u.readkey), "ascii"), file=out)
        print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out)
        print(" fingerprint:", str(base32.b2a(u.fingerprint), "ascii"), file=out)
    elif isinstance(u, uri.SSKVerifierURI):
        if show_header:
            print("SDMF Verifier URI:", file=out)
        print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out)
        print(" fingerprint:", str(base32.b2a(u.fingerprint), "ascii"), file=out)

    elif isinstance(u, uri.WriteableMDMFFileURI): # MDMF
        if show_header:
            print("MDMF Writeable URI:", file=out)
        print(" writekey:", str(base32.b2a(u.writekey), "ascii"), file=out)
        print(" readkey:", str(base32.b2a(u.readkey), "ascii"), file=out)
        print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out)
        print(" fingerprint:", str(base32.b2a(u.fingerprint), "ascii"), file=out)
        print(file=out)
        if nodeid:
            we = hashutil.ssk_write_enabler_hash(u.writekey, nodeid)
            print(" write_enabler:", str(base32.b2a(we), "ascii"), file=out)
            print(file=out)
        _dump_secrets(u.get_storage_index(), secret, nodeid, out)
    elif isinstance(u, uri.ReadonlyMDMFFileURI):
        if show_header:
            print("MDMF Read-only URI:", file=out)
        print(" readkey:", str(base32.b2a(u.readkey), "ascii"), file=out)
        print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out)
        print(" fingerprint:", str(base32.b2a(u.fingerprint), "ascii"), file=out)
    elif isinstance(u, uri.MDMFVerifierURI):
        if show_header:
            print("MDMF Verifier URI:", file=out)
        print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out)
        print(" fingerprint:", str(base32.b2a(u.fingerprint), "ascii"), file=out)

    elif isinstance(u, uri.ImmutableDirectoryURI): # CHK-based directory
        if show_header:
            print("CHK Directory URI:", file=out)
        dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
    elif isinstance(u, uri.ImmutableDirectoryURIVerifier):
        if show_header:
            print("CHK Directory Verifier URI:", file=out)
        dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)

    elif isinstance(u, uri.DirectoryURI): # SDMF-based directory
        if show_header:
            print("Directory Writeable URI:", file=out)
        dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
    elif isinstance(u, uri.ReadonlyDirectoryURI):
        if show_header:
            print("Directory Read-only URI:", file=out)
        dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
    elif isinstance(u, uri.DirectoryURIVerifier):
        if show_header:
            print("Directory Verifier URI:", file=out)
        dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)

    elif isinstance(u, uri.MDMFDirectoryURI): # MDMF-based directory
        if show_header:
            print("Directory Writeable URI:", file=out)
        dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
    elif isinstance(u, uri.ReadonlyMDMFDirectoryURI):
        if show_header:
            print("Directory Read-only URI:", file=out)
        dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)
    elif isinstance(u, uri.MDMFDirectoryURIVerifier):
        if show_header:
            print("Directory Verifier URI:", file=out)
        dump_uri_instance(u._filenode_uri, nodeid, secret, out, False)

    else:
        print("unknown cap type", file=out)

class FindSharesOptions(BaseOptions):
    def getSynopsis(self):
        return "Usage: tahoe [global-options] debug find-shares STORAGE_INDEX NODEDIRS.."

    def parseArgs(self, storage_index_s, *nodedirs):
        from allmydata.util.encodingutil import argv_to_abspath
        self.si_s = storage_index_s
        self.nodedirs = list(map(argv_to_abspath, nodedirs))

    description = """
Locate all shares for the given storage index. This command looks through one
or more node directories to find the shares. It returns a list of filenames,
one per line, for each share file found.

 tahoe debug find-shares 4vozh77tsrw7mdhnj7qvp5ky74 testgrid/node-*

It may be useful during testing, when running a test grid in which all the
nodes are on a local disk. The share files thus located can be counted,
examined (with dump-share), or corrupted/deleted to test checker/repairer.
"""

def find_shares(options):
    """Given a storage index and a list of node directories, emit a list of
    all matching shares to stdout, one per line. For example:

     find-shares.py 44kai1tui348689nrw8fjegc8c ~/testnet/node-*

    gives:

    /home/warner/testnet/node-1/storage/shares/44k/44kai1tui348689nrw8fjegc8c/5
    /home/warner/testnet/node-1/storage/shares/44k/44kai1tui348689nrw8fjegc8c/9
    /home/warner/testnet/node-2/storage/shares/44k/44kai1tui348689nrw8fjegc8c/2
    """
    from allmydata.storage.server import si_a2b, storage_index_to_dir
    from allmydata.util.encodingutil import listdir_unicode, quote_local_unicode_path

    out = options.stdout
    sharedir = storage_index_to_dir(si_a2b(options.si_s.encode("utf-8")))
    for d in options.nodedirs:
        d = os.path.join(d, "storage", "shares", sharedir)
        if os.path.exists(d):
            for shnum in listdir_unicode(d):
                print(quote_local_unicode_path(os.path.join(d, shnum), quotemarks=False), file=out)

    return 0

class CatalogSharesOptions(BaseOptions):
    def parseArgs(self, *nodedirs):
        from allmydata.util.encodingutil import argv_to_abspath
        self.nodedirs = list(map(argv_to_abspath, nodedirs))
        if not nodedirs:
            raise usage.UsageError("must specify at least one node directory")

    def getSynopsis(self):
        return "Usage: tahoe [global-options] debug catalog-shares NODEDIRS.."

    description = """
Locate all shares in the given node directories, and emit a one-line summary
of each share. Run it like this:

 tahoe debug catalog-shares testgrid/node-* >allshares.txt

The lines it emits will look like the following:

 CHK $SI $k/$N $filesize $UEB_hash $expiration $abspath_sharefile
 SDMF $SI $k/$N $filesize $seqnum/$roothash $expiration $abspath_sharefile
 UNKNOWN $abspath_sharefile

This command can be used to build up a catalog of shares from many storage
servers and then sort the results to compare all shares for the same file. If
you see shares with the same SI but different parameters/filesize/UEB_hash,
then something is wrong. The misc/find-share/anomalies.py script may be
useful for purpose.
"""

def call(c, *args, **kwargs):
    # take advantage of the fact that ImmediateReadBucketProxy returns
    # Deferreds that are already fired
    results = []
    failures = []
    d = defer.maybeDeferred(c, *args, **kwargs)
    d.addCallbacks(results.append, failures.append)
    if failures:
        failures[0].raiseException()
    return results[0]

def describe_share(abs_sharefile, si_s, shnum_s, now, out):
    with open(abs_sharefile, "rb") as f:
        prefix = f.read(32)
        if MutableShareFile.is_valid_header(prefix):
            _describe_mutable_share(abs_sharefile, f, now, si_s, out)
        elif ShareFile.is_valid_header(prefix):
            _describe_immutable_share(abs_sharefile, now, si_s, out)
        else:
            print("UNKNOWN really-unknown %s" % quote_output(abs_sharefile), file=out)

def _describe_mutable_share(abs_sharefile, f, now, si_s, out):
    # mutable share
    m = MutableShareFile(abs_sharefile)
    WE, nodeid = m._read_write_enabler_and_nodeid(f)
    data_length = m._read_data_length(f)
    expiration_time = min( [lease.get_expiration_time()
                            for (i,lease) in m._enumerate_leases(f)] )
    expiration = max(0, expiration_time - now)

    share_type = "unknown"
    f.seek(m.DATA_OFFSET)
    version = f.read(1)
    if version == b"\x00":
        # this slot contains an SMDF share
        share_type = "SDMF"
    elif version == b"\x01":
        share_type = "MDMF"

    if share_type == "SDMF":
        f.seek(m.DATA_OFFSET)

        # Read at least the mutable header length, if possible.  If there's
        # less data than that in the share, don't try to read more (we won't
        # be able to unpack the header in this case but we surely don't want
        # to try to unpack bytes *following* the data section as if they were
        # header data).  Rather than 2000 we could use HEADER_LENGTH from
        # allmydata/mutable/layout.py, probably.
        data = f.read(min(data_length, 2000))

        try:
            pieces = unpack_share(data)
        except NeedMoreDataError as e:
            # retry once with the larger size
            size = e.needed_bytes
            f.seek(m.DATA_OFFSET)
            data = f.read(min(data_length, size))
            pieces = unpack_share(data)
        (seqnum, root_hash, IV, k, N, segsize, datalen,
         pubkey, signature, share_hash_chain, block_hash_tree,
         share_data, enc_privkey) = pieces

        print("SDMF %s %d/%d %d #%d:%s %d %s" % \
              (si_s, k, N, datalen,
               seqnum, str(base32.b2a(root_hash), "utf-8"),
               expiration, quote_output(abs_sharefile)), file=out)
    elif share_type == "MDMF":
        fake_shnum = 0
        # TODO: factor this out with dump_MDMF_share()
        class ShareDumper(MDMFSlotReadProxy):
            def _read(self, readvs, force_remote=False, queue=False):
                data = []
                for (where,length) in readvs:
                    f.seek(m.DATA_OFFSET+where)
                    data.append(f.read(length))
                return defer.succeed({fake_shnum: data})

        p = ShareDumper(None, "fake-si", fake_shnum)
        def extract(func):
            stash = []
            # these methods return Deferreds, but we happen to know that
            # they run synchronously when not actually talking to a
            # remote server
            d = func()
            d.addCallback(stash.append)
            return stash[0]

        verinfo = extract(p.get_verinfo)
        (seqnum, root_hash, salt_to_use, segsize, datalen, k, N, prefix,
         offsets) = verinfo
        print("MDMF %s %d/%d %d #%d:%s %d %s" % \
              (si_s, k, N, datalen,
               seqnum, str(base32.b2a(root_hash), "utf-8"),
               expiration, quote_output(abs_sharefile)), file=out)
    else:
        print("UNKNOWN mutable %s" % quote_output(abs_sharefile), file=out)

def _describe_immutable_share(abs_sharefile, now, si_s, out):
    class ImmediateReadBucketProxy(ReadBucketProxy):
        def __init__(self, sf):
            self.sf = sf
            ReadBucketProxy.__init__(self, None, None, "")
        def __repr__(self):
            return "<ImmediateReadBucketProxy>"
        def _read(self, offset, size):
            return defer.succeed(sf.read_share_data(offset, size))

    # use a ReadBucketProxy to parse the bucket and find the uri extension
    sf = ShareFile(abs_sharefile)
    bp = ImmediateReadBucketProxy(sf)

    expiration_time = min(lease.get_expiration_time()
                          for lease in sf.get_leases())
    expiration = max(0, expiration_time - now)

    UEB_data = call(bp.get_uri_extension)
    unpacked = uri.unpack_extension_readable(UEB_data)

    k = unpacked["needed_shares"]
    N = unpacked["total_shares"]
    filesize = unpacked["size"]
    ueb_hash = unpacked["UEB_hash"]

    print("CHK %s %d/%d %d %s %d %s" % (si_s, k, N, filesize,
                                        str(ueb_hash, "utf-8"), expiration,
                                        quote_output(abs_sharefile)), file=out)

def catalog_shares(options):
    from allmydata.util.encodingutil import listdir_unicode, quote_output

    out = options.stdout
    err = options.stderr
    now = time.time()
    for d in options.nodedirs:
        d = os.path.join(d, "storage", "shares")
        try:
            abbrevs = listdir_unicode(d)
        except EnvironmentError:
            # ignore nodes that have storage turned off altogether
            pass
        else:
            for abbrevdir in sorted(abbrevs):
                if abbrevdir == "incoming":
                    continue
                abbrevdir = os.path.join(d, abbrevdir)
                # this tool may get run against bad disks, so we can't assume
                # that listdir_unicode will always succeed. Try to catalog as much
                # as possible.
                try:
                    sharedirs = listdir_unicode(abbrevdir)
                    for si_s in sorted(sharedirs):
                        si_dir = os.path.join(abbrevdir, si_s)
                        catalog_shares_one_abbrevdir(si_s, si_dir, now, out,err)
                except:
                    print("Error processing %s" % quote_output(abbrevdir), file=err)
                    failure.Failure().printTraceback(err)

    return 0

def _as_number(s):
    try:
        return int(s)
    except ValueError:
        return "not int"

def catalog_shares_one_abbrevdir(si_s, si_dir, now, out, err):
    from allmydata.util.encodingutil import listdir_unicode, quote_output

    try:
        for shnum_s in sorted(listdir_unicode(si_dir), key=_as_number):
            abs_sharefile = os.path.join(si_dir, shnum_s)
            assert os.path.isfile(abs_sharefile)
            try:
                describe_share(abs_sharefile, si_s, shnum_s, now,
                               out)
            except:
                print("Error processing %s" % quote_output(abs_sharefile), file=err)
                failure.Failure().printTraceback(err)
    except:
        print("Error processing %s" % quote_output(si_dir), file=err)
        failure.Failure().printTraceback(err)

class CorruptShareOptions(BaseOptions):
    def getSynopsis(self):
        return "Usage: tahoe [global-options] debug corrupt-share SHARE_FILENAME"

    optParameters = [
        ["offset", "o", "block-random", "Specify which bit to flip."],
        ]

    description = """
Corrupt the given share by flipping a bit. This will cause a
verifying/downloading client to log an integrity-check failure incident, and
downloads will proceed with a different share.

The --offset parameter controls which bit should be flipped. The default is
to flip a single random bit of the block data.

 tahoe debug corrupt-share testgrid/node-3/storage/shares/4v/4vozh77tsrw7mdhnj7qvp5ky74/0

Obviously, this command should not be used in normal operation.
"""
    def parseArgs(self, filename):
        self['filename'] = filename

def corrupt_share(options):
    import random
    from allmydata.storage.mutable import MutableShareFile
    from allmydata.storage.immutable import ShareFile
    from allmydata.mutable.layout import unpack_header
    from allmydata.immutable.layout import ReadBucketProxy
    out = options.stdout
    fn = options['filename']
    assert options["offset"] == "block-random", "other offsets not implemented"
    # first, what kind of share is it?

    def flip_bit(start, end):
        offset = random.randrange(start, end)
        bit = random.randrange(0, 8)
        print("[%d..%d):  %d.b%d" % (start, end, offset, bit), file=out)
        f = open(fn, "rb+")
        f.seek(offset)
        d = f.read(1)
        d = bchr(ord(d) ^ 0x01)
        f.seek(offset)
        f.write(d)
        f.close()

    with open(fn, "rb") as f:
        prefix = f.read(32)

        if MutableShareFile.is_valid_header(prefix):
            # mutable
            m = MutableShareFile(fn)
            with open(fn, "rb") as f:
                f.seek(m.DATA_OFFSET)
                # Read enough data to get a mutable header to unpack.
                data = f.read(2000)
            # make sure this slot contains an SMDF share
            assert data[0:1] == b"\x00", "non-SDMF mutable shares not supported"
            f.close()

            (version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize,
             ig_datalen, offsets) = unpack_header(data)

            assert version == 0, "we only handle v0 SDMF files"
            start = m.DATA_OFFSET + offsets["share_data"]
            end = m.DATA_OFFSET + offsets["enc_privkey"]
            flip_bit(start, end)
        else:
            # otherwise assume it's immutable
            f = ShareFile(fn)
            bp = ReadBucketProxy(None, None, '')
            offsets = bp._parse_offsets(f.read_share_data(0, 0x24))
            start = f._data_offset + offsets["data"]
            end = f._data_offset + offsets["plaintext_hash_tree"]
            flip_bit(start, end)

class ReplOptions(BaseOptions):
    def getSynopsis(self):
        return "Usage: tahoe debug repl (OBSOLETE)"

def repl(options):
    print("'tahoe debug repl' is obsolete. Please run 'python' in a virtualenv.", file=options.stderr)
    return 1

DEFAULT_TESTSUITE = 'allmydata'

class TrialOptions(BaseOptions):
    def getSynopsis(self):
        return "Usage: tahoe debug trial (OBSOLETE)"

def trial(config):
    print("'tahoe debug trial' is obsolete. Please run 'tox', or use 'trial' in a virtualenv.", file=config.stderr)
    return 1

def fixOptionsClass(args):
    (subcmd, shortcut, OptionsClass, desc) = args
    class FixedOptionsClass(OptionsClass):
        def getSynopsis(self):
            t = OptionsClass.getSynopsis(self)
            i = t.find("Usage: flogtool ")
            if i >= 0:
                return "Usage: tahoe [global-options] debug flogtool " + t[i+len("Usage: flogtool "):]
            else:
                return "Usage: tahoe [global-options] debug flogtool %s [options]" % (subcmd,)
    return (subcmd, shortcut, FixedOptionsClass, desc)

class FlogtoolOptions(foolscap_cli.Options):
    def __init__(self):
        super(FlogtoolOptions, self).__init__()
        self.subCommands = list(map(fixOptionsClass, self.subCommands))

    def getSynopsis(self):
        return "Usage: tahoe [global-options] debug flogtool COMMAND [flogtool-options]"

    def parseOptions(self, all_subargs, *a, **kw):
        self.flogtool_args = list(all_subargs)
        return super(FlogtoolOptions, self).parseOptions(self.flogtool_args, *a, **kw)

    def getUsage(self, width=None):
        t = super(FlogtoolOptions, self).getUsage(width)
        t += """
The 'tahoe debug flogtool' command uses the correct imports for this instance
of Tahoe-LAFS.

Please run 'tahoe debug flogtool COMMAND --help' for more details on each
subcommand.
"""
        return t

    def opt_help(self):
        print(str(self))
        sys.exit(0)

def flogtool(config):
    sys.argv = ['flogtool'] + config.flogtool_args
    return foolscap_cli.run_flogtool()

class DebugCommand(BaseOptions):
    subCommands = [
        ["dump-share", None, DumpOptions,
         "Unpack and display the contents of a share (uri_extension and leases)."],
        ["dump-cap", None, DumpCapOptions, "Unpack a read-cap or write-cap."],
        ["find-shares", None, FindSharesOptions, "Locate sharefiles in node dirs."],
        ["catalog-shares", None, CatalogSharesOptions, "Describe all shares in node dirs."],
        ["corrupt-share", None, CorruptShareOptions, "Corrupt a share by flipping a bit."],
        ["repl", None, ReplOptions, "OBSOLETE"],
        ["trial", None, TrialOptions, "OBSOLETE"],
        ["flogtool", None, FlogtoolOptions, "Utilities to access log files."],
        ]
    def postOptions(self):
        if not hasattr(self, 'subOptions'):
            raise usage.UsageError("must specify a subcommand")
    synopsis = "COMMAND"

    def getUsage(self, width=None):
        t = BaseOptions.getUsage(self, width)
        t += """\

Please run e.g. 'tahoe debug dump-share --help' for more details on each
subcommand.
"""
        return t

subDispatch = {
    "dump-share": dump_share,
    "dump-cap": dump_cap,
    "find-shares": find_shares,
    "catalog-shares": catalog_shares,
    "corrupt-share": corrupt_share,
    "repl": repl,
    "trial": trial,
    "flogtool": flogtool,
    }

def do_debug(options):
    so = options.subOptions
    so.stdout = options.stdout
    so.stderr = options.stderr
    f = subDispatch[options.subCommand]
    return f(so)

subCommands : SubCommands = [
    ("debug", None, DebugCommand, "debug subcommands: use 'tahoe debug' for a list."),
    ]

dispatch = {
    "debug": do_debug,
    }


# Morris H. 12/10

from vpython import *

# intial setting

sun_mass, sun_r= 1988500E24, 695700*10**3
earth_mass, earth_r, earth_v = 5.9724*10**24, 6371*10**3, 29780
moon_mass, moon_r, moon_v = 0.07346*10**24, 1737*10**3, 1.023*10**3
sun_earth_r, earth_moon_r = 152.10*10**9 , 384400*10**3
G = 6.67408E-11
T = 2360488

scene = canvas(title='Motion of Moon\n',
                   width=1000, height=600, align='left', range=200)

scene.range = sun_earth_r + earth_moon_r*50

earth = sphere(pos=vec(sun_earth_r, 0, 0), texture=textures.earth, up=vec(-tan(23.5/180*pi), 0, 1),
               radius=earth_r, make_trail=True, v=vec(0, earth_v, 0), a=vec(0, 0, 0))

sun = sphere(pos=vec(0,0,0), radius=sun_r*50, color=color.orange, emissive=True, v=vec(0, 0, 0),
             a=vec(0, 0, 0))

# moon = sphere(pos=vec(earth_moon_r+sun_earth_r, 0, 0), radius=moon_r*20, make_trail=False,
#               v=vec(0, moon_v+earth_v, 0), a=vec(0, 0, 0))

moon = sphere(pos=vec(sun_earth_r+earth_moon_r*10, 0, 0), radius=moon_r*2500, make_trail=True,
              color=vec(0.95, 0.94, 0.78))

g1 = graph(title='Moon Position', align='right', xtitle='X(km)', ytitle='Y(km)',
           width=500, height=350)

scene.append_to_caption('\n\nMake sun 50 times bigger and moon 2500 times bigger\n\n\n\n\n\n\n')

moon_p = gdots(graph=g1, color=color.black)
earth_p = gdots(graph=g1, color=color.blue)

t, dt, i = 0, 60*60, 0
while True:
    rate(150)
    earth.rotate(angle=1, axis=vec(-tan(23.5/180*pi), 0, 1))

    earth.a = G*sun_mass*(sun.pos-earth.pos)/mag(sun.pos-earth.pos)**3
    earth.v += earth.a*dt
    earth.pos += earth.v*dt

    moon.pos = vec(earth_moon_r*cos(dt*i*2*pi/T)*10, earth_moon_r*sin(dt*i*2*pi/T)*10, 0) + earth.pos
    i += 1

    moon_p.plot(pos=(moon.pos.x/1000, moon.pos.y/1000))
    earth_p.plot(pos=(earth.pos.x/1000, earth.pos.y/1000))

    t += dt


from sqlalchemy import *
from sqlalchemy.orm import *
import os

DATABASE = os.environ.get('DATABASE_URL')

# setting for db
ENGINE = create_engine(DATABASE, encoding='utf-8', echo=False)

META = MetaData(bind=ENGINE)
META.reflect()

# create session
session = scoped_session(
    # Setting for ORM, auto commit is set to false. Do session.commit to commit
    sessionmaker(autocommit=False, autoflush=False, bind=ENGINE)
)


"""update company table

Revision ID: cb787028944f
Revises: f8b12c9b32fd
Create Date: 2019-06-21 18:52:31.161077

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'cb787028944f'
down_revision = 'f8b12c9b32fd'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('company', sa.Column('addr', sa.String(length=128), nullable=True))
    op.add_column('company', sa.Column('logo', sa.String(length=256), nullable=True))
    op.add_column('company', sa.Column('slogan', sa.String(length=64), nullable=True))
    op.add_column('company', sa.Column('website', sa.String(length=128), nullable=True))
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('company', 'website')
    op.drop_column('company', 'slogan')
    op.drop_column('company', 'logo')
    op.drop_column('company', 'addr')
    # ### end Alembic commands ###


#!/usr/bin/env python3
"""
Run the GitLabQueueMonitor.

It queries the GitLab API for the number of global pending and running jobs. As well as the runner
specific running jobs.

Example usage:

    # Run once in dry-run mode.
    ./main.py --dry-run

    # Run once and exports metrics.
    ./main.py --es-index="$USER-gitlab-queue"

    # Run in server mode.
    ./main.py --server --es-index="$USER-gitlab-queue"

"""
import argparse
import logging
import os
import sys
import time
from datetime import datetime

import beeline
from elasticsearch import Elasticsearch
from gitlab import Gitlab

DFINITY_PROJECT = 22_487_979

class GitLabQueueMonitor:
    """GitLabQueueMonitor polls the GitLab API and exports queueing related metrics."""

    def __init__(self, es_index_prefix, dry_run):
        """
        Initialize the GitLabQueueMonitor.

        Args:
        ----
            es_index_prefix: The prefix to use for the ElasticSearch index.
            dry_run: Just log, but do not export metrics to ElasticSearch.

        """
        if "GITLAB_TOKEN" not in os.environ:
            raise ValueError("could not find GITLAB_TOKEN in env vars")

        self.es = Elasticsearch(["https://elasticsearch.testnet.dfinity.network:443"])
        self.dry_run = dry_run

        self.gl = Gitlab("https://gitlab.com", private_token=os.getenv("GITLAB_TOKEN"))
        self.project = self.gl.projects.get(DFINITY_PROJECT)
        self.es_index_prefix = es_index_prefix

    def _count_jobs_by_scope(self, scope: str):
        data = {}
        jobs = self.project.jobs.list(scope=scope, all=True)

        for job in jobs:
            with beeline.tracer(name="process-jobs"):
                job_context_dict = {f"job.{key}": val for key, val in job.attributes.items()}
                beeline.add_context(job_context_dict)
            data[job.name] = data.get(job.name, 0) + 1

        return (len(jobs), data)

    def _count_runner_jobs_by_status(self, runner, status: str):
        jobs = runner.jobs.list(status=status, all=True)

        return len(jobs)

    def _export_data_to_elastic_search(self, index_infix, data):
        timestamp = datetime.utcnow()

        data["timestamp"] = timestamp

        logging.info(data)

        if not self.dry_run:
            self.es.index(
                f"{self.es_index_prefix}-{index_infix}-{timestamp.year}",
                id=timestamp,
                body=data,
            )

    @beeline.traced(name="queue-monitor-run")
    def process(self):
        """Process queries the GitLab API and exports metrics to ElasticSearch."""
        pending = self._count_jobs_by_scope("pending")
        running = self._count_jobs_by_scope("running")

        data = {}
        data["pending_queue_data"] = {"job_status": "pending", "count": pending[0]}
        self._export_data_to_elastic_search("global", data["pending_queue_data"])

        data["running_queue_data"] = {"job_status": "running", "count": running[0]}
        self._export_data_to_elastic_search("global", data["running_queue_data"])

        data["pending_job_data"] = {}
        data["running_job_data"] = {}
        for key, value in pending[1].items():
            data["pending_job_data"][key] = {
                "status": "pending",
                "job_name": key,
                "count": value,
            }
            self._export_data_to_elastic_search("job-name", data["pending_job_data"][key])

        for key, value in running[1].items():
            data["running_job_data"][key] = {
                "status": "running",
                "job_name": key,
                "count": value,
            }
            self._export_data_to_elastic_search("job-name", data["running_job_data"][key])

        data["runner_data"] = {}

        runners = self.gl.runners.list(all=True)
        for runner in runners:
            with beeline.tracer(name="process-runner"):
                runner_details = self.gl.runners.get(runner.id)
                runner_context_dict = {f"runner.{key}": val for key, val in runner_details.attributes.items()}
                beeline.add_context(runner_context_dict)

                running_jobs = self._count_runner_jobs_by_status(runner, "running")
                logging.info(f"{runner.description}: {running_jobs}")

                data["runner_data"][runner.description] = {
                    "description": runner.description,
                    "running_jobs": running_jobs,
                }
                self._export_data_to_elastic_search("runners", data["runner_data"][runner.description])

        beeline.add_context_field("error", False)
        return data

def main():
    parser = argparse.ArgumentParser()

    parser.add_argument(
        "--server",
        action="store_true",
        help="In server mode loop forever. Otherwise run once",
    )

    parser.add_argument(
        "--es-index",
        type=str,
        help="ElasticSearch index prefix to export to ",
    )
    parser.add_argument(
        "--dry-run",
        action="store_true",
        help="In dry run mode, only log but do not export metrics to ElasticSearch",
    )
    parser.add_argument(
        "--export-period",
        default="60",
        type=int,
        help="Time between export cycles",
    )

    logging.basicConfig(level=logging.INFO)

    args = parser.parse_args()

    honeycomb_token = os.getenv("HONEYCOMB_API_TOKEN")
    if not honeycomb_token:
        sys.exit("No Honeycomb token specified in HONEYCOMB_API_TOKEN env var.")

    beeline.init(
        # Get this via https://ui.honeycomb.io/account after signing up for Honeycomb
        writekey=honeycomb_token,
        # The name of your app is a good choice to start with
        dataset="gitlab-ci-daemons",
        service_name="queue-monitor",
        debug=False,  # if True, data doesn't get sent to Honeycomb
    )

    queue_monitor = GitLabQueueMonitor(args.es_index, args.dry_run)

    if not args.server:
        queue_monitor.process()
        beeline.close()
        return

    last_export_time = time.time()
    while True:
        try:
            next_export_time = last_export_time + args.export_period
            next_export_delay = max(next_export_time - time.time(), 0)

            logging.info(f"next export in {next_export_delay} seconds")
            time.sleep(next_export_delay)
            last_export_time = time.time()

            queue_monitor.process()
        except Exception as e:
            logging.error(e)

    beeline.close()

if __name__ == "__main__":
    main()


from __future__ import absolute_import, division, print_function
__metaclass__ = type

ANSIBLE_METADATA = {'metadata_version': '1.1',
                    'status': ['preview'],
                    'supported_by': 'community'}

DOCUMENTATION = '''
---
module: kernel_blacklist
author:
- Matthias Vogelgesang (@matze)
version_added: '1.4'
short_description: Blacklist kernel modules
description:
    - Add or remove kernel modules from blacklist.
options:
    name:
        description:
            - Name of kernel module to black- or whitelist.
        required: true
    state:
        description:
            - Whether the module should be present in the blacklist or absent.
        choices: [ absent, present ]
        default: present
    blacklist_file:
        description:
            - If specified, use this blacklist file instead of
              C(/etc/modprobe.d/blacklist-ansible.conf).
'''

EXAMPLES = '''
- name: Blacklist the nouveau driver module
  kernel_blacklist:
    name: nouveau
    state: present
'''

import os
import re

from ansible.module_utils.basic import AnsibleModule

class Blacklist(object):
    def __init__(self, module, filename, checkmode):
        self.filename = filename
        self.module = module
        self.checkmode = checkmode

    def create_file(self):
        if not self.checkmode and not os.path.exists(self.filename):
            open(self.filename, 'a').close()
            return True
        elif self.checkmode and not os.path.exists(self.filename):
            self.filename = os.devnull
            return True
        else:
            return False

    def get_pattern(self):
        return r'^blacklist\s*' + self.module + '$'

    def readlines(self):
        f = open(self.filename, 'r')
        lines = f.readlines()
        f.close()
        return lines

    def module_listed(self):
        lines = self.readlines()
        pattern = self.get_pattern()

        for line in lines:
            stripped = line.strip()
            if stripped.startswith('#'):
                continue

            if re.match(pattern, stripped):
                return True

        return False

    def remove_module(self):
        lines = self.readlines()
        pattern = self.get_pattern()

        if self.checkmode:
            f = open(os.devnull, 'w')
        else:
            f = open(self.filename, 'w')

        for line in lines:
            if not re.match(pattern, line.strip()):
                f.write(line)

        f.close()

    def add_module(self):
        if self.checkmode:
            f = open(os.devnull, 'a')
        else:
            f = open(self.filename, 'a')

        f.write('blacklist %s\n' % self.module)

        f.close()

def main():
    module = AnsibleModule(
        argument_spec=dict(
            name=dict(type='str', required=True),
            state=dict(type='str', default='present', choices=['absent', 'present']),
            blacklist_file=dict(type='str')
        ),
        supports_check_mode=True,
    )

    args = dict(changed=False, failed=False,
                name=module.params['name'], state=module.params['state'])

    filename = '/etc/modprobe.d/blacklist-ansible.conf'

    if module.params['blacklist_file']:
        filename = module.params['blacklist_file']

    blacklist = Blacklist(args['name'], filename, module.check_mode)

    if blacklist.create_file():
        args['changed'] = True
    else:
        args['changed'] = False

    if blacklist.module_listed():
        if args['state'] == 'absent':
            blacklist.remove_module()
            args['changed'] = True
    else:
        if args['state'] == 'present':
            blacklist.add_module()
            args['changed'] = True

    module.exit_json(**args)

if __name__ == '__main__':
    main()


# Import the necessary packages.
from picamera.array import PiRGBArray
from picamera import PiCamera
import cv2
import time
import socket

def int2bytes(n):
    result = bytearray()
    while n:
        result.append(n & 0xff)
        n = n >> 8
    return result[::-1]

# Initialize variables.
SERVER_ADDR = "192.168.1.253"
PORT = 5006
WIDTH = 320
HEIGHT = 180

# Initialize the camera and grab a reference to the raw camera capture.
camera = PiCamera()
camera.vflip = True
camera.hflip = True
camera.resolution = (WIDTH, HEIGHT)
camera.framerate = 32
rawCapture = PiRGBArray(camera, size=(WIDTH, HEIGHT))
time.sleep(0.1)  # Allow the camera to warm up.

# Initialize the socket.
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)  # Internet with UDP.
sock.bind((SERVER_ADDR, PORT))

print("Waiting on clients...")

# Capture frames from the camera.
while(True):
    for frame in camera.capture_continuous(rawCapture, format="bgr", use_video_port=True):
        # Grab the raw NumPy array representing the image, then initialize the timestamp and occupied/unoccupied text.
        image = frame.array
        image.flags.writeable = True

        gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
        gray = cv2.resize(gray, (WIDTH, HEIGHT), interpolation=cv2.INTER_LINEAR)

        # Send the image size to the client.
        data, address = sock.recvfrom(8)
        sock.sendto(int2bytes(gray.size), address)

        # Send the image to the client.
        sock.sendto(gray, address)

        # Clear the stream in preparation for the next frame.
        rawCapture.truncate(0)

        # If the 'q' key was pressed, break from the loop.
        key = cv2.waitKey(1) & 0xFF
        if key == ord("q"):
            break

sock.close()
camera.close()


from Implementations.Interfaces.IDeterministicAlgorithm import IDeterministicAlgorithm

class DynamicProgrammingRegular(IDeterministicAlgorithm):

    def __init__(self):
        self.label = 'DP Regular'

    @classmethod
    def run(cls, values, target):
        return cls.subsetSumDP(values, target)

    @classmethod
    def subsetSumDP(cls, vals, T):
        vals = list(filter(lambda v: v <= T, vals))
        OPT = [[None for i in range(T + 1)] for i in range(len(vals) + 1)]
        # Can never reach a target value >= 1, if not allowed to use any values
        for i in range(1, T + 1):
            OPT[0][i] = False
        # Can always reach zero no matter what, because you simply dont use any weight.
        for i in range(len(vals) + 1):
            OPT[i][0] = True

        for i in range(1, len(vals) + 1):
            tempI = OPT[i]
            tempIMinusOne = OPT[i-1]
            for j in range(1, T + 1):
                #           We use the value   Or we pick it, if the target value is greater than or equal
                tempI[j] = tempIMinusOne[j] or (j >= vals[i - 1] and tempIMinusOne[j - vals[i - 1]])

        return [target for target, achieved in enumerate(OPT[len(vals)]) if achieved]


import cv2
import mediapipe as mp
from HandTrackingModule import HandDetector
import pyautogui as py

detector = HandDetector()

mpHands = mp.solutions.hands
hands = mpHands.Hands()
drawTools=mp.solutions.drawing_utils

capture = cv2.VideoCapture(0)
clocX,clocY,plocX,plocY=0,0,0,0
smoothening=7
frameR=100
while True:
    success, img = capture.read()
    lmlist,img=detector.lmlist(img)
    if lmlist:
        fingers, img=detector.fingersUp(img,lmlist,False)
        if (fingers==[0,1,0,0,1]):
            py.screenshot('screenshoting.png')
            py.alert(text='Screenshot taken', title='Successful', button='OK')

    cv2.imshow("video feed",img)
    key = cv2.waitKey(1)
    if (key==27):
        break

capture.release()
cv2.destroyAllWindows()


#!/usr/bin/env python3

'''
The script remembers current layout for an focused window,
and activates it automatically when the same window gets focus again.

Autostart from the i3 config:
  exec_always --no-startup-id ~/scripts/i3-kb-switcher.py

Dependencies:
  xkb-switch (https://github.com/grwlf/xkb-switch)
'''

import i3ipc
import logging
import os
from subprocess import check_output, STDOUT
import threading

POLL_RATE = 2 # seconds
PID_FILE = '/tmp/i3-kb-switcher.pid'
os.environ['DISPLAY'] = ':0'
layouts_map = {}

#logging.basicConfig(filename='/tmp/i3-kb-switcher.log', level=logging.DEBUG)
log = logging.getLogger('kbsw')

def exec(args):
    try:
        return check_output(args, stderr=STDOUT).decode().strip()
    except Exception as e:
        out = e.output.strip()
        log.error(out)
        return out

def on_window_focus(i3, event):
    id = event.container.id
    log.debug('%s got focus', id)
    l = layouts_map.get(id)
    if l != None:
        exec(['xkb-switch', '-s', l])
        log.debug('Set layout: %s', l)

def on_window_close(i3, event):
    id = event.container.id
    del layouts_map[id]
    log.debug('%s is closed', id)

def get_focused_window(windows):
    for w in windows:
        if w.focused:
            return w.id

def remember_layout():
    threading.Timer(POLL_RATE, remember_layout).start()
    windows = i3.get_tree().find_focused().workspace().descendants()
    focused = get_focused_window(windows)
    if focused != None:
        layout = exec('xkb-switch')
        layouts_map[focused] = layout
        log.debug('%s => %s', focused, layout)

def kill_old_copy():
    exec(['bash', '-c', 'kill -9 $(cat {})'.format(PID_FILE)])
    open(PID_FILE, 'w').write(str(os.getpid()))

def init_i3_conn():
    i3_sp = exec(['i3', '--get-socketpath'])
    log.debug('i3 socket path = %s', i3_sp)
    i3 = i3ipc.Connection(auto_reconnect=True, socket_path=i3_sp)
    i3.on('window::focus', on_window_focus)
    i3.on('window::close', on_window_close)
    return i3

###########################################################

i3 = init_i3_conn()
kill_old_copy()
remember_layout()
log.debug('<<< Started >>>')
i3.main()


import dask.dataframe as ddf
from dask_ml.model_selection import train_test_split
import time
import dask.distributed
import xgboost as xgb
import dask.array as da
import numpy as np
import sklearn.metrics as skm
import matplotlib.pyplot as plt
import pickle
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import KFold, RandomizedSearchCV
import os
from statistics import mean
from joblib import Parallel, delayed
from pprint import pprint
from scipy.stats import uniform, truncnorm, randint
import shutil

def plot_roc_curve(fper, tper):
    plt.plot(fper, tper, color='orange', label='ROC')
    plt.plot([0, 1], [0, 1], color='darkblue', linestyle='--')
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('Receiver Operating Characteristic (ROC) Curve')
    plt.legend()
    plt.savefig("ROC_curve_alexa_data.png") #save as png

def KFoldValidation(train_index, test_index):
	X_train, X_test = X[train_index], X[test_index]
	y_train, y_test = y[train_index], y[test_index]
	global model_previous
	model = model_previous
	model = RandomizedSearchCV(model, model_params, n_iter=2, cv=2, random_state=42, scoring = "accuracy")
	model.fit(X_train, y_train)
	model = model.best_estimator_
	model_previous = model
	y_pred = model.predict(X_test)
	global accuracy
	accuracy.append(skm.accuracy_score(y_test, y_pred))
	global f1_score
	f1_score.append(skm.f1_score(y_test, y_pred))
	global recall
	recall.append(skm.recall_score(y_test, y_pred))
	global precision
	precision.append(skm.precision_score(y_test, y_pred))
#	global roc_auc_score
#	roc_auc_score.append(skm.roc_auc_score(y_test, y_pred))
	global confusion
	confusion = confusion + skm.confusion_matrix(y_test, y_pred, labels = [0,1])
	return { "model":model_previous, "accuracy":accuracy, "f1_score":f1_score,"recall":recall, "precision":precision,"confusion":confusion }
	#return {"model":model.best_estimator_, "score":model.best_score_}

if __name__ == "__main__":
	directories = ['preprocessed_instances_for_overall_test']
	if os.path.exists('./models_'+directories[0][13:-5]):
		shutil.rmtree('./models_'+directories[0][13:-5])
	os.mkdir('./models_'+directories[0][13:-5])
	if os.path.exists('./performances_'+directories[0][13:-5]):
		shutil.rmtree('./performances_'+directories[0][13:-5])
	os.mkdir('./performances_'+directories[0][13:-5])
	for directory in directories:
		for filename in os.listdir('../../code_data/'+directory):
			f = os.path.join('../../code_data/'+directory, filename)
			if os.path.isfile(f) :
				print('--- Loading '+filename+' data ...')
				start_time = time.time()
				df = ddf.read_csv(f)
				df = df.sample(frac=1)
				X = df.iloc[:, 3:22].values
				y = df.iloc[:, 22].values
				print("--- Data loaded in  %s seconds ---" % (time.time() - start_time))
				print('--- Chunksizes computing ...')
				start_time = time.time()
				X.compute_chunk_sizes()
				y.compute_chunk_sizes()
				print("--- Chunksizes computed in  %s seconds ---" % (time.time() - start_time))
				kf = KFold(n_splits=2, random_state = 42, shuffle=True)
				print('--- Training ---')
				start_time = time.time()
				model_params = {
				# randomly sample numbers from 4 to 204 estimators
				#'n_estimators': randint(100,110),
				'criterion' : ['gini', 'entropy'],
				#'max_depth':5,
				# normally distributed max_features, with mean .25 stddev 0.1, bounded between 0 and 1
				'max_features': truncnorm(a=0, b=0.6, loc=0.25, scale=0.1),
				# uniform distribution from 0.01 to 0.2 (0.01 + 0.199)
				'min_samples_split': uniform(0.01, 0.199),
				#'n_jobs': -1,
				#'warm_start':True
				}
				accuracy = []
				f1_score = []
				precision = []
				recall = []
#				roc_auc_score = []
				confusion = [[0,0],[0,0]]
				model_previous = RandomForestClassifier(warm_start=True,n_jobs=-1,n_estimators = 200,max_depth = None)
				res = Parallel(n_jobs=os.cpu_count(), require='sharedmem')(delayed(KFoldValidation)(i,j) for i,j in kf.split(X))[0]
				#rf_model = RandomForestClassifier()
				#model = RandomizedSearchCV(rf_model, model_params,n_jobs = os.cpu_count(),  n_iter=100, cv=5, random_state=1)
				#model.fit(X, y)
				print('--- Training done in %s seconds' % (time.time() - start_time))
				pickle.dump(res["model"], open('./models_'+directory[13:-5]+'/rf_'+filename[22:-4]+'.pkl', "wb"))
				#pprint(model.best_estimator_.get_params())
				with open("./performances_"+directory[13:-5]+"/performance_"+filename[22:-4]+".txt", "w") as file1:
					#file1.write('--- Accuracy : '+str(res["score"])+'\n')
					file1.write('--- Accuracy : '+str(mean(accuracy))+'\n')
					file1.write('--- F1 score : '+str(mean(f1_score))+'\n')
					file1.write('--- Precision : '+str(mean(precision))+'\n')
					file1.write('--- Recall : '+str(mean(recall))+'\n')
#					file1.write('--- ROC AUC score : '+str(mean(roc_auc_score))+'\n')
					file1.write('--- Confusion Matrix : '+str(confusion)+'\n')


import numpy as np
import os
import tarfile
from six.moves import urllib
import pandas as pd
from matplotlib import pyplot as plt
import pandas as pd
from sklearn.model_selection import train_test_split, StratifiedShuffleSplit
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import Imputer
from sklearn.preprocessing import StandardScaler
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.pipeline import FeatureUnion

# 定数の定義
DOWNLAD_ROOT = 'https://raw.githubusercontent.com/ageron/handson-ml/master/'
HOUSING_PATH = os.path.join('datasets', 'housing')
HOUSING_NAME = 'housing.tgz'
HOUSING_URL  = DOWNLAD_ROOT + 'datasets/housing/housing.tgz'

def fetch_housing_data(housing_url=HOUSING_URL, housing_path=HOUSING_PATH):
    if not os.path.isdir(housing_path):
        os.makedirs(housing_path)
    tgz_path = os.path.join(housing_path, 'housing.tgz')
    urllib.request.urlretrieve(housing_url, tgz_path)
    housing_tgz = tarfile.open(tgz_path)
    housing_tgz.extractall(path=housing_path)
    housing_tgz.close()

def downloadAfterCheck(housing_url=HOUSING_URL, housing_path=HOUSING_PATH, housing_name=HOUSING_NAME):
    if os.path.isfile(os.path.join(housing_path, housing_name)):
        None
    else:
        fetch_housing_data(housing_url, housing_path)

def load_housing_data(path):
    return pd.read_csv(path)

class DataFrameSelector(BaseEstimator, TransformerMixin):
    def __init__(self, num_columns):
        self.num_columns = num_columns
    def fit(self, X, y=None):
        return self
    def transform(self, X, y=None):
        return X[self.num_columns].values

class OneHotEncoder(BaseEstimator, TransformerMixin):
    def __init__(self):
        None
    def fit(self, X, y=None):
        return self
    def transform(self, X, y=None):
        # 転置行列にする
        T = X.T
        # 返却用の配列を定義
        X_trs = []
        # 文字列属性分、ワンホットエンコードを実施
        for col in np.arange(T.shape[0]):
            # 属性に含まれる値の分、列を追加＆バイナリ化
            for val in set(T[col]):
                tmp = T[col].copy()
                tmp[tmp==val] = 1
                tmp[tmp!=1] = 0
                X_trs.append(tmp)
        # 転置し直す
        X_trs = np.array(X_trs)
        return X_trs.T

class DataFactory():
    def __init__(self):
        self.data          = None
        self.data_train    = None
        self.data_test     = None
        self.data_prepared = None
        self.data_label    = None

    def split_dataset(self, data):
        # 層化分割用の属性を追加
        data['income_cat'] = np.ceil(data['median_income'] / 1.5)
        data['income_cat'].where(data['income_cat'] < 5, 5.0, inplace=True)

        # 層化分割
        split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)
        for train_index, test_index in split.split(data, data['income_cat']):
            self.data_train = data.loc[train_index]
            self.data_test  = data.loc[test_index]
        self.data       = data

        # 層化分割用の属性を削除
        for set_ in (self.data_train, self.data_test):
            set_.drop('income_cat', axis=1, inplace=True)

        # 壊れないようにコピーを渡す
        return self.data_train.copy(), self.data_test.copy()

    def split_label(self, data, label_name):
        data_label = data[label_name].copy()
        data_other = data.drop(label_name, axis=1)

        return data_label, data_other

    def getPreparedDataSet(self):
        if self.data_prepared is None:
            # データのダウンロード
            downloadAfterCheck()

            # データの読込
            housing = load_housing_data('datasets/housing/housing.csv')

            df = DataFactory()
            # 訓練データとテストデータの分割
            data_train, data_test = df.split_dataset(housing)
            # ラベルとデータの分割
            data_label, data_train = df.split_label(data_train, 'median_house_value')

            # 数値属性について実施するパイプラインの定義
            str_columns = ['ocean_proximity']
            num_columns = list(data_train)
            for str_column in str_columns:
                num_columns.remove(str_column)

            pipeline_num = Pipeline([
                ('selector'  , DataFrameSelector(num_columns)),  # 数値属性だけ取り出す
                ('imputer'   , Imputer(strategy='median')),      # 欠損値を中央値で補う
                ('std_scalar', StandardScaler()),                # 標準化する
            ])

            pipeline_str = Pipeline([
                ('selector'   , DataFrameSelector(str_columns)), # 文字列属性だけ取り出す
                ('str_encoder', OneHotEncoder()),                # ワンホットエンコード
            ])

            # ２つのパイプラインで変換したデータを結合させるパイプライン
            pipeline_main = FeatureUnion(transformer_list=[
                ('pipeline_num', pipeline_num),
                ('pipeline_str', pipeline_str),
            ])

            #  パイプラプラインインを実行する
            self.data_prepared = pipeline_main.fit_transform(data_train)

            self.data_label = data_label

        else:
            None
        return self.data_prepared.copy(), self.data_label.copy()


from mrjob.job import MRJob
from mrjob.step import MRStep
import numpy as np

DATE, CURTAIN, VALUE = 0, 3, 1

class MRValueMean(MRJob):

    def steps(self):
        return [
            MRStep(mapper=self.mapper, reducer=self.reducer_mean_month),
            MRStep(mapper=self.mapper_sum_days, reducer=self.reducer_)
        ]

    def mapper(self, _, line):
        line = line.split(',')
        if line[0] != 'StartedOn':
            date, curtain, value = [line[DATE].split('/')[0], line[DATE].split('/')[1]], line[CURTAIN], line[VALUE]
            yield [int(date[0]), int(date[1]), curtain], float(value)

    def reducer_mean_month(self, key, value):
        yield key, np.mean([val for val in value])

    def mapper_sum_days(self, key, value):
        yield (key[1], key[2]), value

    def reducer_(self, key, value):
        values = []
        for val in value:
            values.append(val)
        yield key, round(max(values), 7)
        yield key, round(min(values), 7)

if __name__ == '__main__':
    MRValueMean().run()


# -*- coding: utf-8 -*-

from CuAsm.CuInsAssemblerRepos import CuInsAssemblerRepos
from CuAsm.CuInsFeeder import CuInsFeeder
from CuAsm.common import decodeCtrlCodes, binstr, hexstr

def doExtract(fname, insfilter='', arch='sm_75', maxcnt=100):
    feeder = CuInsFeeder(fname, instfilter=insfilter, arch=arch)

    print("# Searching %s with filter %s:"%(fname, insfilter))
    cnt = 0
    for addr, code, s, ctrl in feeder:
        #s = s.replace('.reuse','')
        ctrlstr = decodeCtrlCodes(ctrl)
        #print(' %s: 0x%06x  0x%032x     %s'%(ctrlstr, addr, code, s))
        print('0x%03x: [%s] %#016x  %s '%(addr, ctrlstr, code, s))
        print(binstr(code, 64))
        print(hexstr(code, 64))

        cnt += 1
        if cnt>=maxcnt:
            break

sassname1 = 'G:\\Temp\\cudnn64_7.sm_50.sass'
sassname2 = 'G:\\Temp\\cudnn64_7.sass'
#sassname1 = 'D:\\MyProjects\\Programs\\cudatest.sm_75.sass'
#sassname2 = 'D:\\MyProjects\\Programs\\cudatest.sm_52.sass'
#sassname = 'G:\\Temp\\cublas64_10.sm_75.sass'
# initialize a feeder with sass
#doExtract(sassname2, r'reuse', arch='sm_75', maxcnt=8)
#print(sassname1)
# doExtract(sassname1, '\.NEG', arch='sm_50', maxcnt=100)
# doExtract(sassname1, '-0\.5', arch='sm_50', maxcnt=100)
doExtract(sassname1, 'FADD.*0\.5', arch='sm_50', maxcnt=10)
#print(sassname2)
#doExtract(sassname2, '', arch='sm_52', maxcnt=888)
#doExtract(sassname1, 'PBK', arch='sm_50', maxcnt=8)
#doExtract(sassname1, 'BRX', arch='sm_50', maxcnt=8)
#
#doExtract(sassname1, 'BRA', arch='sm_50', maxcnt=8)
#doExtract(sassname1, 'CAL', arch='sm_50', maxcnt=8)
#doExtract(sassname1, 'BRK', arch='sm_50', maxcnt=8)
#doExtract(sassname1, 'RET', arch='sm_50', maxcnt=8)
#doExtract(sassname1, 'SSY', arch='sm_50', maxcnt=8)
#doExtract(sassname1, 'SYNC', arch='sm_50', maxcnt=8)
#
#doExtract(sassname2, 'BRA', arch='sm_75', maxcnt=8)
#doExtract(sassname2, 'CAL', arch='sm_75', maxcnt=8)
#doExtract(sassname2, 'BREAK', arch='sm_75', maxcnt=8)
#doExtract(sassname2, 'RET', arch='sm_75', maxcnt=8)
#doExtract(sassname2, 'BSSY', arch='sm_75', maxcnt=8)
#doExtract(sassname2, 'BSYNC', arch='sm_75', maxcnt=8)


#legendre polynomial
#(9) integrate(-1 --> 1)(P(n,x)*P(m,x)) = (2*dmn)/(2*n+1)

import numpy as np
from scipy.special import legendre
from scipy.misc import derivative
from scipy.integrate import quad

def f(x):
    return (legendre(m)(x))*(legendre(n)(x))

x=float(input("Enter the x: "))
m=int(input("Enter the m: "))
n=int(input("Enter the n: "))
LHS=quad(f,-1,1)[0]

if(m==n):
    d=1
    RHS=((2.0*d)/(2*n+1))
    print("(m==n):\n",RHS)
elif(m!=n):
    d=0
    RHS=((2.0*d)/(2*n+1))
    print("(m!=n): ",RHS)
print("LHS:\n",LHS)


from .bulk import bulk
from .errors import ScanError

import logging

logger = logging.getLogger('xapiand.helpers')

def scan(client, query=None, scroll='5m', raise_on_error=True,
         preserve_order=False, size=1000, request_timeout=None, clear_scroll=True,
         scroll_kwargs=None, **kwargs):
    """
    Simple abstraction on top of the
    :meth:`~xapiand.Xapiand.scroll` api - a simple iterator that
    yields all hits as returned by underlining scroll requests.

    By default scan does not return results in any pre-determined order. To
    have a standard order in the returned documents (either by score or
    explicit sort definition) when scrolling, use ``preserve_order=True``. This
    may be an expensive operation and will negate the performance benefits of
    using ``scan``.

    :arg client: instance of :class:`~xapiand.Xapiand` to use
    :arg query: body for the :meth:`~xapiand.Xapiand.search` api
    :arg scroll: Specify how long a consistent view of the index should be
        maintained for scrolled search
    :arg raise_on_error: raises an exception (``ScanError``) if an error is
        encountered (some shards fail to execute). By default we raise.
    :arg preserve_order: don't set the ``search_type`` to ``scan`` - this will
        cause the scroll to paginate with preserving the order. Note that this
        can be an extremely expensive operation and can easily lead to
        unpredictable results, use with caution.
    :arg size: size (per shard) of the batch send at each iteration.
    :arg request_timeout: explicit timeout for each call to ``scan``
    :arg clear_scroll: explicitly calls delete on the scroll id via the clear
        scroll API at the end of the method on completion or error, defaults
        to true.
    :arg scroll_kwargs: additional kwargs to be passed to
        :meth:`~xapiand.Xapiand.scroll`

    Any additional keyword arguments will be passed to the initial
    :meth:`~xapiand.Xapiand.search` call::

        scan(client,
            query={"query": {"match": {"title": "python"}}},
            index="orders-*",
            doc_type="books"
        )

    """
    scroll_kwargs = scroll_kwargs or {}

    if not preserve_order:
        query = query.copy() if query else {}
        query["sort"] = "_doc"
    # initial search
    resp = client.search(body=query, scroll=scroll, size=size,
                         request_timeout=request_timeout, **kwargs)

    scroll_id = resp.get('_scroll_id')
    if scroll_id is None:
        return

    try:
        first_run = True
        while True:
            # if we didn't set search_type to scan initial search contains data
            if first_run:
                first_run = False
            else:
                resp = client.scroll(scroll_id, scroll=scroll,
                                     request_timeout=request_timeout,
                                     **scroll_kwargs)

            for hit in resp['hits']['hits']:
                yield hit

            # check if we have any errrors
            if resp["_shards"]["successful"] < resp["_shards"]["total"]:
                logger.warning(
                    "Scroll request has only succeeded on %d shards out of %d.",
                    resp['_shards']['successful'], resp['_shards']['total']
                )
                if raise_on_error:
                    raise ScanError(
                        scroll_id,
                        "Scroll request has only succeeded on %d shards out of %d." % (
                            resp['_shards']['successful'], resp['_shards']['total']
                        )
                    )

            scroll_id = resp.get('_scroll_id')
            # end of scroll
            if scroll_id is None or not resp['hits']['hits']:
                break
    finally:
        if scroll_id and clear_scroll:
            client.clear_scroll(body={'scroll_id': [scroll_id]}, ignore=(404, ))

def reindex(client, source_index, target_index, query=None, target_client=None,
            chunk_size=500, scroll='5m', scan_kwargs={}, bulk_kwargs={}):
    """
    Reindex all documents from one index that satisfy a given query
    to another, potentially (if `target_client` is specified) on a different cluster.
    If you don't specify the query you will reindex all the documents.

    Since ``2.3`` a :meth:`~xapiand.Xapiand.reindex` api is
    available as part of xapiand itself. It is recommended to use the api
    instead of this helper wherever possible. The helper is here mostly for
    backwards compatibility and for situations where more flexibility is
    needed.

    .. note::

        This helper doesn't transfer mappings, just the data.

    :arg client: instance of :class:`~xapiand.Xapiand` to use (for
        read if `target_client` is specified as well)
    :arg source_index: index (or list of indices) to read documents from
    :arg target_index: name of the index in the target cluster to populate
    :arg query: body for the :meth:`~xapiand.Xapiand.search` api
    :arg target_client: optional, is specified will be used for writing (thus
        enabling reindex between clusters)
    :arg chunk_size: number of docs in one chunk sent to the server (default: 500)
    :arg scroll: Specify how long a consistent view of the index should be
        maintained for scrolled search
    :arg scan_kwargs: additional kwargs to be passed to
        :func:`~xapiand.helpers.scan`
    :arg bulk_kwargs: additional kwargs to be passed to
        :func:`~xapiand.helpers.bulk`
    """
    target_client = client if target_client is None else target_client

    docs = scan(
        client,
        query=query,
        index=source_index,
        scroll=scroll,
        **scan_kwargs
    )

    def _change_doc_index(hits, index):
        for h in hits:
            h['_index'] = index
            if 'fields' in h:
                h.update(h.pop('fields'))
            yield h

    kwargs = {
        'stats_only': True,
    }
    kwargs.update(bulk_kwargs)
    return bulk(target_client,
                _change_doc_index(docs, target_index),
                chunk_size=chunk_size,
                **kwargs)


import numpy as np
from scipy.interpolate import interp1d

class Si2Ai:
    """
    This class transforms a 'Raw signal matrix' into 'Signal Image' and
    then do 2-D Discrete Fourier Transform (DFT)on it.

    Refer to the paper
    'Human Activity Recognition Using Wearable Sensors by Deep Convolutional Neural Networks'
    for more Details

    Raw signal matrix :   a 2D array  which stores permutated raw signals with a shape of
                        Reading type Num(9 for UCI-HAR data set)   * window_size (50Hz mul 2.56s=128)

    Signal Image:         a (Nsis-1) * windows_size matrix

    DFT Image:            has same shape of Signal Image mentioned above

    """

    # TODO: we can build the concatenation rule according to the number of sensor readings so that there is no need
    # TODO: to generate the rule for every Raw signal matrix by invoking the "Raw2Sig" function
    def __init__(self, magnitude=True):
        self.magnitude = magnitude

    def __call__(self, sample):
        # 9*128  # scalr
        reading, label = sample["input"], sample["label"]
        if self.magnitude:
            dft = (np.absolute(np.fft.fft2(Raw2Sig(reading)))).astype(np.float32)
        else:
            dft = (np.fft.fft2(Raw2Sig(reading))).real.astype(np.float32)
        return {"input": dft[np.newaxis, :], "label": label}

def Raw2Sig(Raw):
    Ns = len(Raw)
    i = 1
    j = i + 1
    SI = Raw[i - 1].reshape(1, -1)
    SIS = '%d' % i
    Nsis = 1

    while i != j:
        if j > Ns:
            j = 1
        elif ('%d%d' % (i, j) not in SIS) and ('%d%d' % (j, i) not in SIS):
            SI = np.concatenate((SI, Raw[j - 1].reshape(1, -1)))
            SIS += '%d' % j
            Nsis += 1
            i = j
            j = i + 1
        else:
            j += 1

    return SI[:-1]

class Ucihar4Convlstm:
    """
    Every sample generated by UCIHAR class defined in 'datasets.py' has a reading shaped as

    Batch * sensors(9) * window_size(128).

    However, the ConvLstm model requires an input which is

    Batch  *  1 * L(128 for UCIHar) * D(9 for UCIHar data set).

    To handle this problem, we build this class to do some transform.
    """

    def __call__(self, sample):
        reading, label = sample["input"], sample["label"]
        reading = np.swapaxes(reading, 0, 1).astype(np.float32)
        return {"input": reading[np.newaxis, :], "label": label}

def interpolation(t, y, f_model=50, f_device=50, kind='quadratic'):
    """

    :param t:
    :param y:
    :param f_model:     the sampling frequency of data set used to train the model
    :param f_device:    the sampling frequency of data to be fed into a trained model
    :param kind:        the interpolation type
    :return:            (t_new,y_new)
    """
    kinds = ['zero', 'slinear', 'quadratic', 'cubic', 'linear', 'nearest', 'previous', 'next']
    if kind not in kinds:
        raise TypeError("scipy does not support '{}' interpolation ".format(kind))
    if f_model == f_device:
        return t, y
    L = len(y)

    f = interp1d(t, y, kind=kind)
    Lm = int((f_model / f_device) * L)

    tnew = np.linspace(0, t[-1], num=Lm, endpoint=True)

    return tnew, f(tnew)

class Hapt2Ignatov:
    """
    The shapes of sensor readings stored in 'x_train.npy',"y_train.npy","fea_train.npy" are
    (7767, 128, 6), (7767,) and (7767,40) respectively.
    The input shape of Ignatov model should be N x D(6) x 1 x S(usually128)

    """

    def __call__(self, _sample):
        input, label = _sample["input"], _sample["label"]
        reading, fea_vec = input
        reading = np.swapaxes(reading, 0, 1)
        reading = reading[:, np.newaxis, :]
        return {"input": (reading, fea_vec), "label": label}

if __name__ == "__main__":
    from utils.datasets import UCIHAR
    import matplotlib.pyplot as plt

    ucihar = UCIHAR(transform=Si2Ai(False))

    samples = [ucihar[1], ucihar[2000], ucihar[3000], ucihar[7000]]
    for i in range(len(samples)):
        sample = samples[i]
        dftimg, label = sample['input'].squeeze(), sample['label']
        print(label)
        plt.subplot('22%d' % (i + 1))
        plt.imshow(dftimg.astype(np.uint8), cmap='gray')
        plt.title("Activity Image of %s" % ucihar.activity[label])
    plt.suptitle("DFT of several SignalImages")
    plt.show()


import random
import time
def bubbleSort(arr):
    n = len(arr)
    swapped = False
    for i in range(n-1):
        for j in range(0, n-i-1):
            if arr[j] > arr[j + 1]:
                swapped = True
                arr[j], arr[j + 1] = arr[j + 1], arr[j]
        if not swapped:
            return
start_time = time.time()
rand_list100=[]
rand_list200=[]
rand_list300=[]
a=100
b=200
c=300
for i in range(a):
    rand_list100.append(random.randint(0,20))

for i in range(b):
    rand_list200.append(random.randint(0,20))

for i in range(c):
    rand_list300.append(random.randint(0,20))

bubbleSort(rand_list100)
bubbleSort(rand_list200)
bubbleSort(rand_list300)
print("sekund ",end=" ")
print(time.time() - start_time)


from scipy.integrate import solve_ivp
from scipy.integrate import quad
import numpy as np
import matplotlib.pyplot as plt

Nu = 1.81e-5
Const_vel = 10

def ue(s):
    return Const_vel

def fdUe_ds(s):
    return 0

def re_theta(s):
    return ue(s)*theta(s)/Nu

def theta(s):
    theta2ue6 = np.array([0.45*Nu*quad(lambda ss: ue(ss)**5, 0, ss)[0] for ss in s])
    Theta = np.sqrt(theta2ue6/(ue(s)**6))
    return Theta

def lambbda(s):
    return theta(s)**2 * (1/Nu) * fdUe_ds(s)

def landH(s):
    Lambdas = lambbda(s)
    def test_lambda(Lambda):
        if Lambda <= -0.1:
            l = 0.22 + 1.402*(-0.1) + (0.018*-0.1)/(-0.1+0.107)
            H = 2.088 + 0.0731/(-0.1+0.14)
        elif -0.1 < Lambda <= 0:
            l = 0.22 + 1.402*Lambda + (0.018*Lambda)/(Lambda+0.107)
            H = 2.088 + 0.0731/(Lambda+0.14)
        elif 0 <= Lambda <= 0.1:
            l = 0.22 + 1.57*Lambda - 1.8*(Lambda**2)
            H = 2.61 - 3.75*Lambda + 5.24*(Lambda**2)
        elif Lambda >= 0.1:
            l = 0.22 + 1.57*0.1 - 1.8*0.01
            H = 2.61 - 3.75*0.1 + 5.24*0.01
        return l, H
    return np.array([test_lambda(l) for l in Lambdas])

def cf(s):
    l = landH(s)[:, 0]
    return 2*l/re_theta(s)

def dstar(s):
    H = landH(s)[:, 1]
    return theta(s)*H

def lamtest():
    s = np.linspace(0, 1, 100)
    blasius_d1 = 1.72*np.sqrt(Nu*s/ue(s))

    fig, axs = plt.subplots(2, 2)
    axs[0, 0].plot(s, theta(s))
    axs[0, 0].set_title('Momentum Thickness')
    axs[0, 1].plot(s, cf(s))
    axs[0, 1].set_title('Cf')
    axs[1, 0].plot(s, landH(s)[:, 1])
    axs[1, 0].set_title('H')
    axs[1, 1].plot(s, dstar(s), label="Calculation")
    axs[1, 1].plot(s, blasius_d1, label="Empirical")
    axs[1, 1].set_title('Displacement Thickness')
    axs[1, 1].legend()
    plt.show()