"""Test of the object-store container module."""
import hashlib
import io
import os
import random
import shutil
import tempfile
import uuid
import zlib

import psutil
import pytest

from disk_objectstore import Container
import disk_objectstore.utils as utils
import disk_objectstore.exceptions as exc

class UnopenableBytesIO(io.BytesIO):

    def __enter__(self):
        raise AttributeError('__enter__ method disabled for UnopenableBytesIO')

def _assert_empty_repo(container):
    """Check that the container has no objects and no pack files.

    :param container: a Container.
    """
    counts = container.count_objects()
    assert counts['packed'] == 0, (
        'The container should be empty at the beginning '
        '(but there are {} packed objects)'.format(counts['packed'])
    )
    assert counts['loose'] == 0, (
        'The container should be empty at the beginning '
        '(but there are {} loose objects)'.format(counts['loose'])
    )
    assert counts['pack_files'] == 0, (
        'The container should be empty at the beginning '
        '(but there are {} pack files)'.format(counts['pack_files'])
    )

def _add_objects_loose_loop(container, data):
    """Add loose objects to a container, within a for loop.

    :param container: a Container
    :param data: a dictionary where the key is an arbitrary identifier, and the value is a byte string.
    :return: a dictionary where the keys are the object UUIDs as returned by the container, and the values
      are the keys in the original ``data`` dictionary.
    """
    retval = {}
    for key, content in data.items():
        obj_uuid = container.add_object(content)
        retval[obj_uuid] = key
    return retval

def _add_objects_directly_to_pack(container, data, compress):
    """Add objects directly to pack files.

    :param container: a Container
    :param data: a dictionary where the key is an arbitrary identifier, and the value is a byte string.
    :return: a dictionary where the keys are the object UUIDs as returned by the container, and the values
      are the keys in the original ``data`` dictionary.
    :param compress: whether to use compression or not.
    """
    # Store sorted lists of keys and values
    keys = list(data.keys())
    values = [data[key] for key in keys]

    obj_uuids = container.add_objects_to_pack(values, compress=compress)

    return dict(zip(obj_uuids, keys))

def _get_data_and_md5_loop(container, obj_uuids):
    """Get the MD5 of the data stored under the given container, one at a time in a loop.

    :param container: a Container
    :param obj_uuids: a list of object UUIDs
    :return: a dictionary where the keys are the object UUIDs and the values are the MD5 hexdigests.
    """
    retval = {}
    for obj_uuid in obj_uuids:
        retrieved_content = container.get_object_content(obj_uuid)
        retval[obj_uuid] = hashlib.md5(retrieved_content).hexdigest()
    return retval

def _get_data_and_md5_bulk(container, obj_uuids):
    """Get the MD5 of the data stored under the given container as a single bulk operation.

    :param container: a Container
    :param obj_uuids: a list of object UUIDs
    :return: a dictionary where the keys are the object UUIDs and the values are the MD5 hexdigests.
    """
    retval = {}
    retrieved_contents = container.get_object_contents(obj_uuids)
    for obj_uuid in retrieved_contents:
        retval[obj_uuid] = hashlib.md5(retrieved_contents[obj_uuid]).hexdigest()
    return retval

@pytest.mark.parametrize('retrieve_bulk', [True, False])
def test_add_get_loose(temp_container, generate_random_data, retrieve_bulk):
    """Add a number of objects (one by one, loose) to the container.

    Then retrieve them and check the content is correct."""
    _assert_empty_repo(temp_container)

    data = generate_random_data()

    # Store
    obj_md5s = _add_objects_loose_loop(temp_container, data)

    counts = temp_container.count_objects()
    assert counts['packed'] == 0, (
        'The container should have no packed objects '
        '(but there are {} instead)'.format(counts['packed'])
    )
    assert counts['loose'] == len(obj_md5s), (
        'The container should have {} loose objects '
        '(but there are {} instead)'.format(len(obj_md5s), counts['loose'])
    )

    # Retrieve objects (loose), in random order
    random_keys = list(obj_md5s.keys())
    random.shuffle(random_keys)

    # Retrieve data in a loop
    if retrieve_bulk:
        retrieved_md5s = _get_data_and_md5_bulk(temp_container, random_keys)
    else:
        retrieved_md5s = _get_data_and_md5_loop(temp_container, random_keys)

    # Check that the keys are the same
    assert set(obj_md5s) == set(retrieved_md5s)
    # Check that the MD5 are correct
    for obj_uuid in obj_md5s:
        assert obj_md5s[obj_uuid] == retrieved_md5s[obj_uuid], "Object '{}' has wrong MD5s ({} vs {})".format(
            obj_uuid, obj_md5s[obj_uuid], retrieved_md5s[obj_uuid]
        )

@pytest.mark.parametrize('use_compression,retrieve_bulk', [(True, True), (True, False), (False, True), (False, False)])
def test_add_get_with_packing(temp_container, generate_random_data, use_compression, retrieve_bulk):
    """Add a number of objects (one by one, loose) to the container.

    Then retrieve them and check the content is correct."""
    _assert_empty_repo(temp_container)

    data = generate_random_data()

    # Store
    obj_md5s = _add_objects_loose_loop(temp_container, data)

    # Pack all loose objects
    temp_container.pack_all_loose(compress=use_compression)

    counts = temp_container.count_objects()
    assert counts['packed'] == len(obj_md5s), (
        'The container should have {} packed objects '
        '(but there are {} instead)'.format(len(obj_md5s), counts['packed'])
    )
    assert counts['loose'] == 0, (
        'The container should have 0 loose objects '
        '(but there are {} instead)'.format(counts['loose'])
    )

    # Retrieve objects (loose), in random order
    random_keys = list(obj_md5s.keys())
    random.shuffle(random_keys)

    # Retrieve data in a loop
    if retrieve_bulk:
        retrieved_md5s = _get_data_and_md5_bulk(temp_container, random_keys)
    else:
        retrieved_md5s = _get_data_and_md5_loop(temp_container, random_keys)

    # Check that the keys are the same
    assert set(obj_md5s) == set(retrieved_md5s)
    # Check that the MD5 are correct
    for obj_uuid in obj_md5s:
        assert obj_md5s[obj_uuid] == retrieved_md5s[obj_uuid], "Object '{}' has wrong MD5s ({} vs {})".format(
            obj_uuid, obj_md5s[obj_uuid], retrieved_md5s[obj_uuid]
        )

@pytest.mark.parametrize('use_compression', [True, False])
def test_directly_to_pack_content(temp_container, generate_random_data, use_compression):
    """Add a number of objects directly to packs.

    Then retrieve them and check the content is correct (always bulk retrieve for simplicity)."""
    _assert_empty_repo(temp_container)

    data = generate_random_data()

    obj_md5s = _add_objects_directly_to_pack(temp_container, data, compress=use_compression)

    counts = temp_container.count_objects()
    assert counts['packed'] == len(data), (
        'The container should have {} packed objects '
        '(but there are {} instead)'.format(len(data), counts['packed'])
    )
    assert counts['loose'] == 0, (
        'The container should have 0 loose objects '
        '(but there are {} instead)'.format(counts['loose'])
    )

    # Retrieve objects (loose), in random order
    random_keys = list(obj_md5s.keys())
    random.shuffle(random_keys)

    # Retrieve data in a loop
    retrieved_md5s = _get_data_and_md5_bulk(temp_container, random_keys)

    # Check that the keys are the same
    assert set(obj_md5s) == set(retrieved_md5s)
    # Check that the MD5 are correct
    for obj_uuid in obj_md5s:
        assert obj_md5s[obj_uuid] == retrieved_md5s[obj_uuid], "Object '{}' has wrong MD5s ({} vs {})".format(
            obj_uuid, obj_md5s[obj_uuid], retrieved_md5s[obj_uuid]
        )

@pytest.mark.parametrize('use_compression,open_streams', [(True, True), (True, False), (False, True), (False, False)])
def test_directly_to_pack_streamed(temp_container, generate_random_data, use_compression, open_streams):
    """Add a number of objects directly to packs, using streams.

    Then retrieve them and check the content is correct (always bulk retrieve for simplicity)."""
    _assert_empty_repo(temp_container)

    data = generate_random_data()

    # Store sorted lists of keys and values
    keys = list(data.keys())
    values = [data[key] for key in keys]

    if open_streams:
        streams = []
        streams_copy = []
        with tempfile.TemporaryDirectory() as temp_dir:
            # Read a given fixed order, otherwise later when reconstructing obj_md5s we would have a problem
            for key, content in zip(keys, values):
                file_path = os.path.join(temp_dir, key)
                with open(file_path, 'bw') as fhandle:
                    fhandle.write(content)
                streams.append(utils.LazyOpener(file_path, mode='rb'))
                streams_copy.append(utils.LazyOpener(file_path, mode='rb'))
            obj_uuids = temp_container.add_streamed_objects_to_pack(
                streams, compress=use_compression, open_streams=True
            )
            # I check that instead it fails if I forget to open the streams (and that it does not create side effects)
            # The error that I get here would be: "LazyOpener' object has no attribute 'read'"
            with pytest.raises(AttributeError):
                temp_container.add_streamed_objects_to_pack(streams_copy, compress=use_compression, open_streams=False)

    else:
        streams = [UnopenableBytesIO(value) for value in values]
        streams_copy = [UnopenableBytesIO(value) for value in values]
        obj_uuids = temp_container.add_streamed_objects_to_pack(streams, compress=use_compression, open_streams=False)
        # I check that instead it fails if I try to open but I am not allowed to
        # (and that it does not create side effects). Note that I disabled explicitly the __enter__ method
        # in the class UnopenableBytesIO to make sure that this raises
        with pytest.raises(AttributeError):
            temp_container.add_streamed_objects_to_pack(streams_copy, compress=use_compression, open_streams=True)

    obj_md5s = dict(zip(obj_uuids, keys))

    counts = temp_container.count_objects()
    assert counts['packed'] == len(data), (
        'The container should have {} packed objects '
        '(but there are {} instead)'.format(len(data), counts['packed'])
    )
    assert counts['loose'] == 0, (
        'The container should have 0 loose objects '
        '(but there are {} instead)'.format(counts['loose'])
    )

    # Retrieve objects (loose), in random order
    random_keys = list(obj_md5s.keys())
    random.shuffle(random_keys)

    # Retrieve data in a loop
    retrieved_md5s = _get_data_and_md5_bulk(temp_container, random_keys)

    # Check that the keys are the same
    assert set(obj_md5s) == set(retrieved_md5s)
    # Check that the MD5 are correct
    for obj_uuid in obj_md5s:
        assert obj_md5s[obj_uuid] == retrieved_md5s[obj_uuid], "Object '{}' has wrong MD5s ({} vs {})".format(
            obj_uuid, obj_md5s[obj_uuid], retrieved_md5s[obj_uuid]
        )

@pytest.mark.parametrize('loose_prefix_len,pack_prefix_len', [(0, 2), (2, 2), (3, 2), (0, 3), (2, 3), (3, 3)])
def test_prefix_lengths(temp_dir, generate_random_data, pack_prefix_len, loose_prefix_len):
    """Check if the prefix lengths are honored."""
    container = Container(temp_dir)
    container.init_container(clear=True, pack_prefix_len=pack_prefix_len, loose_prefix_len=loose_prefix_len)
    # Check that the `get_folder` method returns the expected folder name
    assert container.get_folder() == os.path.realpath(temp_dir)

    assert container.loose_prefix_len == loose_prefix_len
    assert container.pack_prefix_len == pack_prefix_len

    _assert_empty_repo(container)
    data = generate_random_data()

    # Store
    obj_md5s = _add_objects_loose_loop(container, data)

    loose_firstlevel = os.listdir(container._get_loose_folder())  # pylint: disable=protected-access
    assert len(loose_firstlevel) > 0
    if loose_prefix_len == 0:
        assert all(len(inode) == 32 for inode in loose_firstlevel)
    else:
        assert all(len(inode) == loose_prefix_len for inode in loose_firstlevel)

    counts = container.count_objects()
    assert counts['packed'] == 0, (
        'The container should have 0 packed objects '
        '(but there are {} instead)'.format(counts['packed'])
    )
    assert counts['loose'] == len(obj_md5s), (
        'The container should have {} loose objects '
        '(but there are {} instead)'.format(len(obj_md5s), counts['loose'])
    )

    retrieved_md5s = _get_data_and_md5_bulk(container, obj_md5s.keys())
    # Check that the MD5 are correct
    for obj_uuid in obj_md5s:
        assert obj_md5s[obj_uuid] == retrieved_md5s[obj_uuid], "Object '{}' has wrong MD5s ({} vs {})".format(
            obj_uuid, obj_md5s[obj_uuid], retrieved_md5s[obj_uuid]
        )

    # Pack all loose objects
    container.pack_all_loose()

    pack_firstlevel = os.listdir(container._get_pack_folder())  # pylint: disable=protected-access
    assert len(pack_firstlevel) > 0
    assert all(len(inode) == pack_prefix_len for inode in pack_firstlevel)

    counts = container.count_objects()
    assert counts['packed'] == len(obj_md5s), (
        'The container should have {} packed objects '
        '(but there are {} instead)'.format(len(obj_md5s), counts['packed'])
    )
    assert counts['loose'] == 0, (
        'The container should have 0 loose objects '
        '(but there are {} instead)'.format(counts['loose'])
    )

    retrieved_md5s = _get_data_and_md5_bulk(container, obj_md5s.keys())
    # Check that the MD5 are correct
    for obj_uuid in obj_md5s:
        assert obj_md5s[obj_uuid] == retrieved_md5s[obj_uuid], "Object '{}' has wrong MD5s ({} vs {})".format(
            obj_uuid, obj_md5s[obj_uuid], retrieved_md5s[obj_uuid]
        )

    # Test also the validation functions
    valid_pack_ids = ['0' * pack_prefix_len, 'a' * pack_prefix_len]
    invalid_pack_ids = ['0' * (pack_prefix_len + 1), 'a' * (pack_prefix_len + 1), 'g' * pack_prefix_len]
    valid_loose_prefixes = ['0' * loose_prefix_len, 'a' * loose_prefix_len]
    invalid_loose_prefixes = [
        '0' * (loose_prefix_len + 1), 'a' * (loose_prefix_len + 1), 'g' * loose_prefix_len if loose_prefix_len else 'g'
    ]

    for pack_id in valid_pack_ids:
        assert container._is_valid_pack_id(  # pylint: disable=protected-access
            pack_id
        ), "'{}' should be valid".format(pack_id)
    for pack_id in invalid_pack_ids:
        assert not container._is_valid_pack_id(  # pylint: disable=protected-access
            pack_id
        ), "'{}' should be invalid".format(pack_id)
    for loose_prefix in valid_loose_prefixes:
        assert container._is_valid_loose_prefix(  # pylint: disable=protected-access
            loose_prefix
        ), "'{}' should be valid".format(loose_prefix)
    for loose_prefix in invalid_loose_prefixes:
        assert not container._is_valid_loose_prefix(  # pylint: disable=protected-access
            loose_prefix
        ), "'{}' should be invalid".format(loose_prefix)

    # I close the container, as this is needed on Windows
    container.close()

@pytest.mark.parametrize('loose_prefix_len,pack_prefix_len', [(-1, 2), (2, -1), (2, 0)])
def test_invalid_prefix_lengths(temp_dir, pack_prefix_len, loose_prefix_len):
    """Check if the prefix lengths are honored."""
    container = Container(temp_dir)
    with pytest.raises(ValueError):
        container.init_container(clear=True, pack_prefix_len=pack_prefix_len, loose_prefix_len=loose_prefix_len)

def test_initialisation(temp_dir):
    """Test that the initialisation function works as expected."""
    container = Container(temp_dir)
    assert not container.is_initialised

    with pytest.raises(exc.NotInitialised):
        _ = container.loose_prefix_len
    with pytest.raises(exc.NotInitialised):
        _ = container.pack_prefix_len

    # Check that the session cannot be obtained before initialising
    with pytest.raises(FileNotFoundError):
        container._get_session(create=False, raise_if_missing=True)  # pylint: disable=protected-access
    assert container._get_session(create=False, raise_if_missing=False) is None  # pylint: disable=protected-access

    container.init_container()
    assert container.is_initialised

    #This call should go through
    container.init_container(clear=True)
    assert container.is_initialised

    with pytest.raises(FileExistsError) as excinfo:
        container.init_container()
    assert 'already exists' in str(excinfo.value)

    # I artificially remove one of the folders: it should notice and say it's not initialised
    os.rmdir(os.path.join(container.get_folder(), 'sandbox'))
    assert not container.is_initialised

    # Close open files (the DB)
    container.close()

    # Remove the folder
    shutil.rmtree(temp_dir)
    assert not container.is_initialised
    # Make back the folder: I leave temp_dir in a consistent state, and moreover I
    # check that the empty folder is again considered non-initialised
    os.mkdir(temp_dir)
    assert not container.is_initialised

    # Create an empty file
    with open(os.path.join(container.get_folder(), 'somefile'), 'w'):
        pass
    # Final re-initialisation
    with pytest.raises(FileExistsError) as excinfo:
        container.init_container()
    assert 'already some file or folder' in str(excinfo.value)

# Only three options: if pack_objects is False, the values of compress_packs is ignored
@pytest.mark.parametrize('pack_objects,compress_packs', [(True, True), (True, False), (False, False)])
def test_unknown_uuids(temp_container, generate_random_data, pack_objects, compress_packs):
    """Put some data in the container, then check that unknown UUIDs raise the correct error."""
    # Generate and store some data, just not to have an empty container
    data = generate_random_data()
    obj_md5s = _add_objects_loose_loop(temp_container, data)

    if pack_objects:
        temp_container.pack_all_loose(compress=compress_packs)

    # Pick any valid UUID
    obj_uuids = list(obj_md5s.keys())

    # 5 unknown UUIDs + one invalid string
    unknown_uuids = [uuid.uuid4().hex for _ in range(5)] + ['invalid-uuid-string']

    # Loop read
    for unknown_uuid in unknown_uuids:
        with pytest.raises(exc.NotExistent):
            temp_container.get_object_content(unknown_uuid)

        # Currently, the exception is actually raised when accessing the stream,
        # not at stream creation
        stream = temp_container.get_object_stream(unknown_uuid)
        with pytest.raises(exc.NotExistent):
            with stream:
                pass

    # 6 invalid UUIDs + all valid ones
    uuids_list = unknown_uuids + obj_uuids
    # I shuffle so they are really in random order
    random.shuffle(uuids_list)

    ##### Bulk reads from here on
    # skip_if_missing=True, get contents
    contents = temp_container.get_object_contents(uuids_list, skip_if_missing=True)
    # The retrieved values should be only the valid ones
    assert set(contents.keys()) == set(obj_uuids)
    check_md5s = {key: hashlib.md5(val).hexdigest() for key, val in contents.items()}
    assert obj_md5s == check_md5s

    # skip_if_missing=True, get streams
    missing = []
    check_md5s = {}
    with temp_container.get_object_streams_and_size(uuids_list, skip_if_missing=True) as triplets:
        for obj_uuid, stream, size in triplets:
            if stream is None:
                assert size == 0
                missing.append(obj_uuid)
            else:
                check_md5s[obj_uuid] = stream.read()
                assert len(check_md5s[obj_uuid]) == size
    # The retrieved values should be only the valid ones
    assert missing == []
    check_md5s = {key: hashlib.md5(val).hexdigest() for key, val in contents.items()}
    assert obj_md5s == check_md5s

    # skip_if_missing=False, get objects
    contents = temp_container.get_object_contents(uuids_list, skip_if_missing=False)
    # There should be only one return value
    for unknown_uuid in unknown_uuids:
        # Check that it's there, that it's noe, and pop it
        assert contents.pop(unknown_uuid) is None
    # After popping, I should be left with the same case as above
    assert set(contents.keys()) == set(obj_uuids)
    check_md5s = {key: hashlib.md5(val).hexdigest() for key, val in contents.items()}
    assert obj_md5s == check_md5s

    # skip_if_missing=False, get streams
    missing = []
    check_md5s = {}
    with temp_container.get_object_streams_and_size(uuids_list, skip_if_missing=False) as triplets:
        for obj_uuid, stream, size in triplets:
            if stream is None:
                assert size == 0
                missing.append(obj_uuid)
            else:
                check_md5s[obj_uuid] = stream.read()
                assert len(check_md5s[obj_uuid]) == size
    # The retrieved values should be only the valid ones
    assert set(missing) == set(unknown_uuids)
    check_md5s = {key: hashlib.md5(val).hexdigest() for key, val in contents.items()}
    assert obj_md5s == check_md5s

@pytest.mark.parametrize('compress_packs', [True, False])
def test_sizes(temp_container, generate_random_data, compress_packs):
    """Check that the information on size is reliable."""
    size_info = temp_container.get_total_size()
    assert size_info['total_size_packed'] == 0
    assert size_info['total_size_packed_on_disk'] == 0
    assert size_info['total_size_packfiles_on_disk'] == 0
    assert size_info['total_size_packindexes_on_disk'] == os.path.getsize(temp_container._get_pack_index_path())  # pylint: disable=protected-access
    assert size_info['total_size_loose'] == 0

    data = generate_random_data()
    total_object_size = sum(len(value) for value in data.values())
    obj_md5s = _add_objects_loose_loop(temp_container, data)
    # Try to count size after retrieving, just to be sure
    assert sum(
        len(content) for content in temp_container.get_object_contents(obj_md5s.keys()).values()
    ) == total_object_size

    # Check the size for loose objects
    size_info = temp_container.get_total_size()
    assert size_info['total_size_packed'] == 0
    assert size_info['total_size_packed_on_disk'] == 0
    assert size_info['total_size_packfiles_on_disk'] == 0
    assert size_info['total_size_packindexes_on_disk'] == os.path.getsize(temp_container._get_pack_index_path())  # pylint: disable=protected-access
    assert size_info['total_size_loose'] == total_object_size

    # Pack without compression
    temp_container.pack_all_loose(compress=compress_packs)
    # Try to count size after retrieving, just to be sure
    assert sum(
        len(content) for content in temp_container.get_object_contents(obj_md5s.keys()).values()
    ) == total_object_size

    if compress_packs:
        # Compress data manually to get compressed size
        # In the current version, compression level is hardcoded.
        # If this becomes a parameter, we need to change this test
        # Note that until when we want to support py3.5, we cannot specify the
        # level as a keyword argument, as this was added only in python 3.6
        compressed_data = {
            key: zlib.compress(val, temp_container._COMPRESSLEVEL)  # pylint: disable=protected-access
            for key, val in data.items()
        }
        total_compressed_size = sum(len(value) for value in compressed_data.values())

        size_info = temp_container.get_total_size()
        assert size_info['total_size_packed'] == total_object_size
        assert size_info['total_size_packed_on_disk'] == total_compressed_size
        assert size_info['total_size_packfiles_on_disk'] == total_compressed_size
        assert size_info['total_size_packindexes_on_disk'] == os.path.getsize(temp_container._get_pack_index_path())  # pylint: disable=protected-access
        assert size_info['total_size_loose'] == 0
    else:
        size_info = temp_container.get_total_size()
        assert size_info['total_size_packed'] == total_object_size
        assert size_info['total_size_packed_on_disk'] == total_object_size
        assert size_info['total_size_packfiles_on_disk'] == total_object_size
        assert size_info['total_size_packindexes_on_disk'] == os.path.getsize(temp_container._get_pack_index_path())  # pylint: disable=protected-access
        assert size_info['total_size_loose'] == 0

def test_get_object_streams_closes(temp_container, generate_random_data):
    """Test that get_object_streams_and_size closes intermediate streams."""
    data = generate_random_data()
    # Store
    obj_md5s = _add_objects_loose_loop(temp_container, data)

    # I get all objects first - this will actually internally go through the same function
    # `get_object_streams_and_size`, but I need to do it as this might open additional files,
    # namely the SQLite DB (possibly more than one file due to the fact it's open in WAL mode).
    # The following checks are still meaningful, I check that if I do it again I don't open more files.
    temp_container.get_object_contents(obj_md5s.keys())

    current_process = psutil.Process()
    start_open_files = len(current_process.open_files())

    with temp_container.get_object_streams_and_size(obj_md5s.keys(), skip_if_missing=True):
        # I don't use the triplets
        pass
    # Check that at the end nothing is left open
    assert len(current_process.open_files()) == start_open_files

    print(current_process.open_files())
    with temp_container.get_object_streams_and_size(obj_md5s.keys(), skip_if_missing=True) as triplets:
        # I loop over the triplets, but I don't do anything
        for _ in triplets:
            pass
    # Check that at the end nothing is left open
    print(current_process.open_files())
    assert len(current_process.open_files()) == start_open_files

    # I actually read the content
    with temp_container.get_object_streams_and_size(obj_md5s.keys(), skip_if_missing=True) as triplets:
        # I loop over the triplets, but I don't do anything
        for _, stream, _ in triplets:
            stream.read()
    # Check that at the end nothing is left open
    assert len(current_process.open_files()) == start_open_files

    ##### Same test after packing
    temp_container.pack_all_loose()
    # I get all objects first, again - this is because it might have closed the DB files while packing
    temp_container.get_object_contents(obj_md5s.keys())
    # I now update the count
    start_open_files = len(current_process.open_files())

    with temp_container.get_object_streams_and_size(obj_md5s.keys()):
        # I don't use the triplets
        pass
    # Check that at the end nothing is left open
    assert len(current_process.open_files()) == start_open_files

    with temp_container.get_object_streams_and_size(obj_md5s.keys()) as triplets:
        # I loop over the triplets, but I don't do anything
        for _ in triplets:
            pass
    # Check that at the end nothing is left open
    assert len(current_process.open_files()) == start_open_files

    # I actually read the content
    with temp_container.get_object_streams_and_size(obj_md5s.keys(), skip_if_missing=True) as triplets:
        # I loop over the triplets, but I don't do anything
        for _, stream, _ in triplets:
            stream.read()
    # Check that at the end nothing is left open
    assert len(current_process.open_files()) == start_open_files


# 使用方式类似于AC自动机:
# KMP(pattern)：构造函数, pattern为模式串.
# Match(s,start): 返回模式串在s中出现的所有位置.
# Move(pos, char): 从当前状态pos沿着char移动到下一个状态, 如果不存在则移动到fail指针指向的状态.
# IsMatched(pos): 判断当前状态pos是否为匹配状态.
# Period(i): 求字符串 s 的前缀 s[:i+1] 的最短周期(0<=i<n). 如果不存在周期, 返回0.

# https://www.ruanyifeng.com/blog/2013/05/Knuth%E2%80%93Morris%E2%80%93Pratt_algorithm.html

from typing import List, Optional

class KMP:
    """单模式串匹配"""

    @staticmethod
    def getNext(pattern: str) -> List[int]:
        next = [0] * len(pattern)
        j = 0
        for i in range(1, len(pattern)):
            while j and pattern[i] != pattern[j]:
                j = next[j - 1]
            if pattern[i] == pattern[j]:
                j += 1
            next[i] = j
        return next

    __slots__ = ("next", "_pattern")

    def __init__(self, pattern: str):
        self._pattern = pattern
        self.next = self.getNext(pattern)

    def match(self, s: str, start=0) -> List[int]:
        res = []
        pos = 0
        for i in range(start, len(s)):
            pos = self.move(pos, s[i])
            if self.isMatched(pos):
                res.append(i - len(self._pattern) + 1)
                pos = 0
        return res

    def move(self, pos: int, input_: str) -> int:
        assert 0 <= pos < len(self._pattern)
        while pos and input_ != self._pattern[pos]:
            pos = self.next[pos - 1]  # rollback
        if input_ == self._pattern[pos]:
            pos += 1
        return pos

    def isMatched(self, pos: int) -> bool:
        return pos == len(self._pattern)

    def period(self, i: Optional[int] = None) -> int:
        """
        求字符串 s 的前缀 s[:i+1] 的最短周期(0<=i<n)
        如果不存在周期, 返回0.
        """
        if i is None:
            i = len(self._pattern) - 1
        assert 0 <= i < len(self._pattern)
        res = (i + 1) - self.next[i]
        if res and (i + 1) > res and (i + 1) % res == 0:
            return res
        return 0

def getNext(needle: str) -> List[int]:
    """kmp O(n)求 `needle`串的 `next`数组
    `next[i]`表示`[:i+1]`这一段字符串中最长公共前后缀(不含这一段字符串本身,即真前后缀)的长度
    """
    next = [0] * len(needle)
    j = 0
    for i in range(1, len(needle)):
        while j and needle[i] != needle[j]:  # 1. fallback后前进：匹配不成功j往右走
            j = next[j - 1]
        if needle[i] == needle[j]:  # 2. 匹配：匹配成功j往右走一步
            j += 1
        next[i] = j
    return next

if __name__ == "__main__":
    next = getNext("aabaabaabaab")  # 模式串的next数组
    assert next == [0, 1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9]

    kmp = KMP("aab")
    assert kmp.match("aabaabaabaab") == [0, 3, 6, 9]
    assert kmp.match("aabaabaabaab", 1) == [3, 6, 9]

    pos = 0
    nextPos = kmp.move(pos, "a")
    assert nextPos == 1
    nextPos = kmp.move(nextPos, "a")
    assert nextPos == 2
    nextPos = kmp.move(nextPos, "b")
    assert kmp.isMatched(nextPos)


from functools import reduce

class Solution:
    def singleNumbers(self, nums: list) -> list:
        ret = reduce(lambda x, y: x ^ y, nums)
        div = 1
        while div & ret == 0:
            div <<= 1
        a, b = 0, 0
        for num in nums:
            if num & div:
                a ^= num
            else:
                b ^= num
        return [a, b]

nums = [4, 1, 4, 6]
test = Solution()
print(test.singleNumbers(nums))


# -*- coding: utf-8 -*-

# Import required libraries
import os
from random import randint

import plotly.plotly as py

import flask
import dash
from dash.dependencies import Input, Output, State, Event
import dash_core_components as dcc
import dash_html_components as html
import plotly.graph_objs as go
import os
from datetime import datetime as dt
import base64
import pandas

# Setup the app
# Make sure not to change this file name or the variable names below,
# the template is configured to execute 'server' on 'app.py'
server = flask.Flask(__name__)
server.secret_key = os.environ.get('secret_key', str(randint(0, 1000000)))
app = dash.Dash(__name__, server=server)

colors = {
    'background': '#999999',
    'text': '#7FDBFF'
}

all_options = {
    'Invoice': ['Concrete', 'Landscaping', 'Site Work'],
    'Estimate': ['Concrete', 'Landscaping', 'Site Work']
}

def create_logo():
	return dcc.Graph(
        id='life-exp-vs-gdp',
        figure={
            'data': [
                go.Scatter(
                    x=[],
                    y=[],
                    text=[],
                    mode='markers',
                    opacity=0.7,
                    marker={
                        'size': 15,
                        'line': {'width': 0.5, 'color': 'white'}
                    },
                )
            ],
            'layout': go.Layout(
            	images=[dict(
        			source="https://raw.githubusercontent.com/CullenBoldt/msystems/master/logo.png",
        			xref="paper", yref="paper",
        			x=.4, y=-.7,
        			sizex=2, sizey=2,
        			xanchor="center", yanchor="bottom")],
        		autosize=False, height=100, width=165,
        		xaxis={},
            	yaxis={},
            	margin={'l': 40, 'b': 40, 't': 10, 'r': 10},
            	legend={'x': 0, 'y': 1},
           		hovermode='closest'

            )

    })

app.layout = html.Div(style={'backgroundColor': colors['background']}, children=[

	html.Div(style={'backgroundColor': colors['background']},children=[      # page 1

    	html.H2(children="MSystems Concrete Estimate", style={'width': '100%', 'padding': '10px 0px 0px 0px'}),
    	html.Hr(),

        html.Div(style={'backgroundColor': colors['background']},children=[ # row 1

    		# Field for cost per yard of concrete
    		html.Div([
    			html.H4(children='''
        				Concrete:
    					''',className = "three columns" ),
    			html.Div([
    				html.Div(children='''
        				Volume concrete:
    					'''),

    			dcc.Input(
    				id='concrete_volume',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),

    			html.Div([
    				html.Div(children='''
        				$/yard concrete:
    					'''),
    			dcc.Input(
    				id='concrete_price',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),
    		],className = "row" ),
    		html.Hr(),

    	], className = "row" ),   #end subpage 1

		html.Div(style={'backgroundColor': colors['background']},children=[ # row 2

    		# Field for cost per yard of Backfill
    		html.Div([
    			html.H4(children='''
        				Backfill:
    					''',className = "three columns" ),
    			html.Div([
    				html.Div(children='''
        				Volume backfill:
    					'''),

    			dcc.Input(
    				id='backfill_volume',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),

    			html.Div([
    				html.Div(children='''
        				$/square foot backfill:
    					'''),
    			dcc.Input(
    				id='backfill_price',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),
    		],className = "row" ),
    		html.Hr(),

    	], className = "row" ),     #end row 2

    	html.Div(style={'backgroundColor': colors['background']},children=[ # row 3

    		# Field for cost per yard of Base
    		html.Div([
    			html.H5(children='''
        				Base:
    					''',className = "three columns" ),
    			html.Div([
    				html.Div(children='''
        				Quantity base:
    					'''),

    			dcc.Input(
    				id='base_volume',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),

    			html.Div([
    				html.Div(children='''
        				$/ton base:
    					'''),
    			dcc.Input(
    				id='base_price',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),
    		],className = "row" ),
    		html.Hr(),

    	], className = "row" ),     #end row 3

    	html.Div(style={'backgroundColor': colors['background']},children=[ # row 4

    		# Field for cost trucking base
    		html.Div([
    			html.H5(children='''
        				Trucking:
    					''',className = "three columns" ),
    			html.Div([
    				html.Div(children='''
        				Quantity base:
    					'''),

    			dcc.Input(
    				id='trucking_volume',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),

    			html.Div([
    				html.Div(children='''
        				$/ton base trucking:
    					'''),
    			dcc.Input(
    				id='trucking_price',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),
    		],className = "row" ),
    		html.Hr(),

    	], className = "row" ),     #end row 4

    	html.Div(style={'backgroundColor': colors['background']},children=[ # row 6

    		# Field for cost trucking base
    		html.Div([
    			html.H4(children='''
        				Finish:
    					''',className = "three columns" ),
    			html.Div([
    				html.Div(children='''
        				Square feet:
    					'''),

    			dcc.Input(
    				id='finish_volume',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),

    			html.Div([
    				html.Div(children='''
        				$/square foot:
    					'''),
    			dcc.Input(
    				id='finish_price',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),
    		],className = "row" ),
    		html.Hr(),

    	], className = "row" ),     #end row 6

    	html.Div(style={'backgroundColor': colors['background']},children=[ # row 6.5

    		# Field for cost trucking base
    		html.Div([
    			html.H4(children='''
        				Form Set:
    					''',className = "three columns" ),
    			html.Div([
    				html.Div(children='''
        				Square feet:
    					'''),

    			dcc.Input(
    				id='form_volume',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),

    			html.Div([
    				html.Div(children='''
        				$/square foot:
    					'''),
    			dcc.Input(
    				id='form_price',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),
    		],className = "row" ),
    		html.Hr(),

    	], className = "row" ),     #end row 6.5

    	html.Div(style={'backgroundColor': colors['background']},children=[ # row 7

    		# Field for cost trucking base
    		html.Div([
    			html.H5(children='''
        				Steel Tying:
    					''',className = "three columns" ),
    			html.Div([
    				html.Div(children='''
        				Square feet:
    					'''),

    			dcc.Input(
    				id='tying_volume',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),

    			html.Div([
    				html.Div(children='''
        				$/square foot:
    					'''),
    			dcc.Input(
    				id='tying_price',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),
    		],className = "row" ),
    		html.Hr(),

    	], className = "row" ),     #end row 7

    	html.Div(style={'backgroundColor': colors['background']},children=[ # row 7.5

    		# Field for cost trucking base
    		html.Div([
    			html.H5(children='''
        				Plaster:
    					''',className = "three columns" ),
    			html.Div([
    				html.Div(children='''
        				Square feet:
    					'''),

    			dcc.Input(
    				id='plaster_volume',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),

    			html.Div([
    				html.Div(children='''
        				$/square foot:
    					'''),
    			dcc.Input(
    				id='plaster_price',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),
    		],className = "row" ),
    		html.Hr(),

    	], className = "row" ),     #end row 7.5

    	html.Div(style={'backgroundColor': colors['background']},children=[ # row 8

    		# Field for cost trucking base
    		html.Div([
    			html.H5(children='''
        				Cable and Steel:
    					''',className = "two columns" ),
    			html.Div([
    				html.Div(children='''
        				Job price:
    					'''),

    			dcc.Input(
    				id='cable_price',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "four columns" ),

    			html.H5(children='''
        				Bagging/Cover:
    					''',className = "three columns" ),

    			html.Div([
    				html.Div(children='''
        				Job Price:
    					'''),
    			dcc.Input(
    				id='poly_price',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "three columns" ),
    		],className = "row" ),
    		html.Hr(),

    	], className = "row" ),     #end row 5

    	html.Div(style={'backgroundColor': colors['background']},children=[ # row 8

    		# Field for cost trucking base
    		html.Div([
    			html.H4(children='''
        				Pump Truck:
    					''',className = "two columns" ),
    			html.Div([
    				html.Div(children='''
        				Job Price:
    					'''),

    			dcc.Input(
    				id='truck_price',
    				placeholder='0.00',
    				type='text',
    				value='1'),
    			],className = "four columns" ),

    			html.H4(children='''
        				Profit:
    					''',className = "two columns" ),
    			html.Div([
    				html.Div(children='''
        				Percentage: e.g.   15 = 15%
    					'''),

    			dcc.Input(
    				id='profit',
    				placeholder='0.00',
    				type='text',
    				value='0'),
    			],className = "three columns" ),

    		],className = "row" ),
    		html.Hr(),

    	], className = "row" ),     #end row 8

    ], className = "page" ), # end page 1

    html.Div(style={'backgroundColor': colors['background']},children=[  # page 2
    	html.A([ 'Print PDF' ],
			className="button no-print",
    		style=dict(position="absolute", top=-40, right=0)),

    	html.Div([
    		html.Div([create_logo()],className = "three column" ),
    		#html.H4(children='''Concrete Estimate:''',className = "five column" ),
    		#html.H3(children="MSystems Concrete Estimate", style={'width': '50%', 'padding': '100px 0px 0px 0px'}),
    	],  className = "row" ),

		html.Div([
			#html.H4(children='''Concrete Estimate:'''),
			html.Div(id='total_out')

		],  className = "row" ), # end final row

    	html.Div([
        	html.Table(id='total_out', className = "w3-table-all w3-tiny"),
            ]),

        html.Div([
        	dcc.Graph(id='pie',config={'displayModeBar': False})], style={'width': '100%', 'padding': '0px 0px 80px 1px','backgroundColor': colors['background']},
            ),

    	html.Div(children='''*This estimate is a best professional assessment including the cost of
    	hiring any subcontractors, the price of materials, and any other labor involved.'''),

    ], className = "page" ), # end page 2

])

#best professional assessment – including the cost of hiring any subcontractors, the price of materials, and any other labor involved.

#Turn key Form setting:  Backfill,  Steel tying, Finishing, Base Material, Concrete

#Cable packages and steel, Bagging poly and cover poly,

@app.callback(
    Output(component_id='total_out', component_property='children'),
    [Input(component_id='concrete_price', component_property='value'),
    Input(component_id='concrete_volume', component_property='value'),
    Input(component_id='backfill_price', component_property='value'),
    Input(component_id='backfill_volume', component_property='value'),
    Input(component_id='base_price', component_property='value'),
    Input(component_id='base_volume', component_property='value'),
    Input(component_id='trucking_price', component_property='value'),
    Input(component_id='trucking_volume', component_property='value'),
    Input(component_id='cable_price', component_property='value'),
    Input(component_id='poly_price', component_property='value'),
    Input(component_id='finish_price', component_property='value'),
    Input(component_id='finish_volume', component_property='value'),
    Input(component_id='form_price', component_property='value'),
    Input(component_id='form_volume', component_property='value'),
    Input(component_id='tying_price', component_property='value'),
    Input(component_id='tying_volume', component_property='value'),
    Input(component_id='plaster_price', component_property='value'),
    Input(component_id='plaster_volume', component_property='value'),
    Input(component_id='truck_price', component_property='value'),
    Input(component_id='profit', component_property='value')]
)
def generate_table(conp,conv,backp,backv,basep,basev,truckbp,truckbv,cable,poly,finishp,finishv,formp,formv,tyingp,tyingv,plasterp,plasterv,truckp,profit):

	dict = {'Concrete' : float(conp)*float(conv), 'Backfill' : float(backp)*float(backv),
			'Base Material' : float(basep)*float(basev), 'Base Trucking': float(truckbv)*float(truckbp),
			'Cable' : float(cable), 'Poly' : float(poly), 'Finish': float(finishp)*float(finishv),
			'Form': float(formp)*float(formv), 'Tying':float(tyingp)*float(tyingv),
			'Plaster':float(plasterp)*float(plasterv), 'Pump Truck' :float(truckp)}

	profit_margin = float(profit) * .01

	for i in dict.keys():
		dict[i] = dict[i]/(1-profit_margin)

	total = format(round(sum(dict.values()),2),',')

	for i in dict.keys():
		dict[i] = str(format(round(dict[i],2), ','))

	table = []
	for row in dict.items():
		html_row = []
		for i in range(len(row)):
			html_row.append( html.Td([ row[i] ]) )
		table.append( html.Tr( html_row, style={'height': '5'}))

	table.append(html.Tr([html.Td([ "Total Estimate"]), html.Td([total])]))
	return table

@app.callback(
    Output(component_id='pie', component_property='figure'),
    [Input(component_id='concrete_price', component_property='value'),
    Input(component_id='concrete_volume', component_property='value'),
    Input(component_id='backfill_price', component_property='value'),
    Input(component_id='backfill_volume', component_property='value'),
    Input(component_id='base_price', component_property='value'),
    Input(component_id='base_volume', component_property='value'),
    Input(component_id='trucking_price', component_property='value'),
    Input(component_id='trucking_volume', component_property='value'),
    Input(component_id='cable_price', component_property='value'),
    Input(component_id='poly_price', component_property='value'),
    Input(component_id='finish_price', component_property='value'),
    Input(component_id='finish_volume', component_property='value'),
    Input(component_id='form_price', component_property='value'),
    Input(component_id='form_volume', component_property='value'),
    Input(component_id='tying_price', component_property='value'),
    Input(component_id='tying_volume', component_property='value'),
    Input(component_id='plaster_price', component_property='value'),
    Input(component_id='plaster_volume', component_property='value'),
    Input(component_id='truck_price', component_property='value'),
    Input(component_id='profit', component_property='value')]
)
def update_pie(conp,conv,backp,backv,basep,basev,truckbp,truckbv,cable,poly,finishp,finishv,formp,formv,tyingp,tyingv,plasterp,plasterv,truckp,profit):

	price_dict = {'Concrete' : float(conp)*float(conv), 'Backfill' : float(backp)*float(backv),
			'Base Material' : float(basep)*float(basev), 'Base Trucking': float(truckbv)*float(truckbp),
			'Cable' : float(cable), 'Poly' : float(poly), 'Finish': float(finishp)*float(finishv),
			'Form': float(formp)*float(formv), 'Tying':float(tyingp)*float(tyingv),
			'Plaster':float(plasterp)*float(plasterv), 'Pump Truck' :float(truckp)}

	profit_margin = float(profit) * .01

	for i in price_dict.keys():
		price_dict[i] = round(price_dict[i]/(1-profit_margin),2)

	layout_pie = {}
	data = [
		dict(
            type='pie',
            labels=[i for i in price_dict.keys()],
            values=[i for i in price_dict.values()],
            name='Production Breakdown',
            hoverinfo="label+value+percent",
            textinfo="label+percent+name",
            rotation = 270,
            hole=0 ,
            marker=dict(
                colors=['#fac1b7', '#a9bb95', '#92d8d8']
            ),
            domain={"x": [0, .8], 'y':[0, 1]},
        	)
	]
	#layout_pie['images'] = [dict(
        #source="https://raw.githubusercontent.com/CullenBoldt/msystems/master/logo.png",
        #xref="paper", yref="paper",
        #x=0.1, y=1,
        #sizex=0.4, sizey=0.4,
        #xanchor="center", yanchor="bottom"
      #)]
	layout_pie['font'] = dict(color='#000000')
	layout_pie['legend'] = dict(font=dict(color='#000000', size='15'), orientation='v', bgcolor='rgba(0,0,0,0)')
	layout_pie['paper_bgcolor']= '#999999'

	figure = dict(data=data, layout= layout_pie)
	return figure

if 'DYNO' in os.environ:
    app.scripts.append_script({
        'external_url': 'https://cdn.rawgit.com/chriddyp/ca0d8f02a1659981a0ea7f013a378bbd/raw/e79f3f789517deec58f41251f7dbb6bee72c44ab/plotly_ga.js'
    })

external_css = [ "https://cdnjs.cloudflare.com/ajax/libs/normalize/7.0.0/normalize.min.css",
        "https://cdnjs.cloudflare.com/ajax/libs/skeleton/2.0.4/skeleton.min.css",
        "//fonts.googleapis.com/css?family=Raleway:400,300,600",
        "https://cdn.rawgit.com/plotly/dash-app-stylesheets/5047eb29e4afe01b45b27b1d2f7deda2a942311a/goldman-sachs-report.css",
        "https://maxcdn.bootstrapcdn.com/font-awesome/4.7.0/css/font-awesome.min.css",
        "https://www.w3schools.com/w3css/4/w3.css"]

for css in external_css:
    app.css.append_css({ "external_url": css })

external_js = [ "https://code.jquery.com/jquery-3.2.1.min.js",
        "https://cdn.rawgit.com/plotly/dash-app-stylesheets/a3401de132a6d0b652ba11548736b1d1e80aa10d/dash-goldman-sachs-report-js.js" ]

for js in external_js:
    app.scripts.append_script({ "external_url": js })

# Run the Dash app
if __name__ == '__main__':
    app.server.run(debug=True, threaded=True)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('web', '0003_auto_20150626_1558'),
    ]

    operations = [
        migrations.AddField(
            model_name='product_web',
            name='secondarymenu',
            field=models.ForeignKey(verbose_name=b'\xe6\x89\x80\xe5\xb1\x9e\xe6\xac\xa1\xe7\xba\xa7\xe8\x8f\x9c\xe5\x8d\x95', blank=True, to='web.Secondarymenu', null=True),
            preserve_default=True,
        ),
        migrations.AlterField(
            model_name='product_web',
            name='submenu',
            field=models.ManyToManyField(to='web.Submenu', null=True, verbose_name=b'\xe6\xac\xa1\xe7\xba\xa7\xe8\x8f\x9c\xe5\x8d\x95\xe5\x88\x86\xe7\xbb\x84', blank=True),
            preserve_default=True,
        ),
    ]


# !/usr/bin/python
# -*- coding: utf-8 -*-
from lab2 import to_json, from_json

__author__ = 'asaskevich'

def main():
    obj = [{'a': 1, 'b': [
        [[1, 2, 3, [[5, 6, [False, [True], [{}, {}, {}, None, False, True, 10.12, -10.13123123]],
                     {'a': {'b': {'c': {'d': [-1, -2, -3, 0]}}}}]]]]]}, {}, {}, False, "sample text",
           {'!': [{}, [{'"': 'this \t is \n quote key'}, {}]]}]
    json = to_json(obj)
    parsed = from_json(json)
    print(json)
    print(parsed)
    print('obj == parsed is', parsed == obj)

if __name__ == '__main__':
    main()


from machine import I2C
import math
from array import array

class HMC5883L():

    __scales = {
        "0.88": [0, 0.73],
        "1.3": [1, 0.92],
        "1.9": [2, 1.22],
        "2.5": [3, 1.52],
        "4.0": [4, 2.27],
        "4.7": [5, 2.56],
        "5.6": [6, 3.03],
        "8.1": [7, 4.35]}

    def __init__(self, i2c, address=30, gauss="1.3", declination=(0,0)):
        self.bus = i2c
        self.address = address
        degrees, minutes = declination
        self.__declDegrees = degrees
        self.__declMinutes = minutes
        self.__declination = (degrees + minutes / 60) * math.pi / 180
        reg, self.__scale = self.__scales[gauss]
        self.bus.writeto_mem(self.address, 0x00, bytearray([0x70])) # 8 Average, 15 Hz, normal measurement
        self.bus.writeto_mem(self.address, 0x01, bytearray([reg << 5])) # Scale
        self.bus.writeto_mem(self.address, 0x02, bytearray([0x00])) # Continuous measurement

    def declination(self):
        return (self.__declDegrees, self.__declMinutes)

    def bytes_toint(self, firstbyte, secondbyte):
        if not firstbyte & 0x80:
            return firstbyte << 8 | secondbyte
        return - (((firstbyte ^ 255) << 8) | (secondbyte ^ 255) + 1)

    def twos_complement(self, val, len): # Convert two's complement to integer
        if (val & (1 << len - 1)):
            val = val - (1<<len)
        return val

    def __convert(self, data, offset):
        x = data[offset] << 8
        #print(x)
        y = data[offset + 1]
        #print(y)
        z = x | y
        #print(z)
        #val = self.twos_complement(z, 16)
        val = self.bytes_toint(data[offset], data[offset + 1])
        if val == -4096:
            #print(val)
            return None
        return round(val * self.__scale, 4)

    def axes(self):
        #data = array('B', [0]*6)
        data = self.bus.readfrom_mem(self.address, 0x03, 6) #Reading just the necessary registers instead of the whole memory as it was in rm-hull's version
        #print(data)
        x = self.__convert(data, 0)
        y = self.__convert(data, 4)
        z = self.__convert(data, 2)
        #print(x, y, z)
        return (x,y,z)

    def heading(self):
        (x, y, z) = self.axes()
        headingRad = math.atan2(y, x)
        headingRad += self.__declination
        # Correct for reversed heading
        if headingRad < 0:
            headingRad += 2 * math.pi
        # Check for wrap and compensate
        elif headingRad > 2 * math.pi:
            headingRad -= 2 * math.pi
        # Convert to degrees from radians
        headingDeg = headingRad * 180 / math.pi
        return headingDeg

    def degrees(self, headingDeg):
        degrees = math.floor(headingDeg)
        minutes = round((headingDeg - degrees) * 60)
        return (degrees, minutes)

    def __str__(self):
        (x, y, z) = self.axes()
        return "Axis X: " + str(x) + " " \
               "Axis Y: " + str(y) + " " \
               "Axis Z: " + str(z) + " " \
               "Heading: " + str(self.heading()) + "\n"

#"Declination: " + self.degrees(self.declination()) + "\n" \ #It gives an error on MicroPython and I've yet to see how it's supposed to work at all


import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
import torch.backends.cudnn as cudnn
import torch.utils.data
import torch.utils.data.distributed
from torch.utils.data.dataloader import default_collate
from tensorboardX import SummaryWriter
import sys
import os
import argparse
from lib.config.hrnet_config import update_config
from lib.config.hrnet_config import config
from lib.models.Hrnet import HigherResolutionNet
from lib.datasets.trip_dataloader import trip_retrieval
from pathlib import Path
# loss
from lib.utils.loss import triplet_loss_cl
from lib.models.Vit_header import VitEncoder

class retrieval_net(nn.Module):
    def __init__(self,cfg, is_train = True, is_transform=True):
        super(retrieval_net,self).__init__()
        self.backbone = HigherResolutionNet(cfg, is_train=is_train)
        self.is_transform = is_transform
        if self.is_transform:
            self.self_attention = VitEncoder(cfg.MODEL_EXTRA.STAGE4.NUM_CHANNELS[0])
    def forward(self, images):
        features = self.backbone(images)
        batch_size = features.shape[0]
        if self.is_transform:
            output_feature = self.self_attention(features)
        else:
            extracted_feature = nn.functional.adaptive_avg_pool2d(features,(1,1))
            output_feature = extracted_feature.reshape(batch_size,-1)
        # need to normalize the feature
        output_feature = output_feature / torch.norm(output_feature,dim=-1,keepdim=True)
        return output_feature

def parse_args():
    parser = argparse.ArgumentParser(description='Train keypoints network')
    parser.add_argument(
        '--cfg', help='experiment configure file name', required=True, type=str)
    parser.add_argument(
        '--transform', help='whether using transform', default=False ,type=bool)
    args, rest = parser.parse_known_args()
    update_config(args.cfg) # 把config的文件更新过去
    return args

def get_optimizer(model):
    lr = config.TRAIN.LR
    optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.module.parameters()), lr=lr) # 整体模型权重均全部重新训练
    return model, optimizer

def load_backbone(model,pretrained_file):
    pretrained_state_dict = torch.load(pretrained_file)
    model_state_dict_backbone = model.module.backbone.state_dict()
    prefix_b = 'backbone.'
    new_pretrained_state_dict_bacbone = {}
    for k, v in pretrained_state_dict.items():
        if k.replace(prefix_b, "") in model_state_dict_backbone and v.shape == model_state_dict_backbone[k.replace(prefix_b, "")].shape:     #.replace(prefix, "") .replace(prefix, "")
            new_pretrained_state_dict_bacbone[k.replace(prefix_b, "")] = v
    print("load statedict from {}".format(pretrained_file))
    model.module.backbone.load_state_dict(new_pretrained_state_dict_bacbone)
    return model

def load_checkpoint(model, optimizer, output_dir, filename='checkpoint.pth.tar'):
    file = os.path.join(output_dir, filename)
    if os.path.isfile(file):
        checkpoint = torch.load(file)
        start_epoch = checkpoint['epoch']
        metrics = checkpoint['loss']
        model.module.load_state_dict(checkpoint['state_dict'])
        optimizer.load_state_dict(checkpoint['optimizer'])
        print('=> load checkpoint {} (epoch {})'
              .format(file, start_epoch))

        return start_epoch, model, optimizer, metrics

    else:
        print('=> no checkpoint found at {}'.format(file))
        return 0, model, optimizer, np.inf

def save_checkpoint(states, is_best, output_dir, filename='checkpoint.pth.tar'):
    torch.save(states, os.path.join(output_dir, filename))
    if is_best and 'state_dict' in states:
        torch.save(states['state_dict'],
                   os.path.join(output_dir, 'model_best.pth.tar'))

def main():
    args = parse_args() # 读取 cfg 参数，config表示之后需要看一下
    result_log_dir = Path(config.OUTPUT_DIR)
    result_log_dir.mkdir(parents=True, exist_ok=True)

    gpus = [int(i) for i in config.GPUS.split(',')]
    image_folder = '/Extra/panzhiyu/img_retrieval/shopee-product-matching/train_images'
    train_dataset = trip_retrieval(image_folder,is_train = True)
    test_dataset = trip_retrieval(image_folder,is_train = False)
    train_loader = torch.utils.data.DataLoader(
        train_dataset,
        batch_size=config.TRAIN.BATCH_SIZE * len(gpus),
        shuffle=config.TRAIN.SHUFFLE,
        num_workers=config.WORKERS,
        pin_memory=True)

    test_loader = torch.utils.data.DataLoader(
        test_dataset,
        batch_size=config.TEST.BATCH_SIZE * len(gpus),
        shuffle=True,
        num_workers=config.WORKERS,
        pin_memory=True)

    cudnn.benchmark = config.CUDNN.BENCHMARK
    torch.backends.cudnn.deterministic = config.CUDNN.DETERMINISTIC
    torch.backends.cudnn.enabled = config.CUDNN.ENABLED
    print('=> Constructing models ..')
    model = retrieval_net(config, is_train= True, is_transform=args.transform)
    with torch.no_grad():
        model = torch.nn.DataParallel(model, device_ids=gpus).cuda()
    model, optimizer = get_optimizer(model)
    start_epoch = config.TRAIN.BEGIN_EPOCH
    end_epoch = config.TRAIN.END_EPOCH
    least_test_loss = np.inf # enough large

    if config.NETWORK.PRETRAINED_BACKBONE: # no pretrained test
        print(f'Using backbone {config.NETWORK.PRETRAINED_BACKBONE}')
        model = load_backbone(model, config.NETWORK.PRETRAINED_BACKBONE) # load POSE ESTIMATION BACKBONE
    if config.TRAIN.RESUME:
        start_epoch, model, optimizer, metrics_load = load_checkpoint(model, optimizer, config.OUTPUT_DIR) # TODO: Load the A1 metrics
        least_test_loss = metrics_load

    tb_log_dir = Path(os.path.join(config.OUTPUT_DIR,'tensorboard_log'))
    tb_log_dir.mkdir(parents=True, exist_ok=True)

    writer_dict = {
        'writer': SummaryWriter(log_dir=str(tb_log_dir)),
        'train_global_steps': 0,
        'valid_global_steps': 0,
    }

    print('=> Training...')
    device=torch.device('cuda')
    for epoch in range(start_epoch, end_epoch):
        print('Epoch: {}'.format(epoch))
        train_sim_loss = AverageMeter()
        test_sim_loss = AverageMeter()
        trip_class_loss = triplet_loss_cl()
        # The train part
        model.train()
        for i, batch in enumerate(train_loader):
            # import pdb; pdb.set_trace()
            q_images, g_images = batch
            if q_images.shape[0] == 1:
                continue # cannot do the triplet loss
            q_images = q_images.to(device)
            g_images = g_images.to(device)

            q_features = model(q_images)
            g_features = model(g_images)

            # calculate the loss as triplet
            trip_loss = trip_class_loss(q_features,g_features)
            train_sim_loss.update(trip_loss.item())
            optimizer.zero_grad()
            trip_loss.backward()
            optimizer.step()

            if i % config.PRINT_FREQ == 0:
                gpu_memory_usage = torch.cuda.memory_allocated(0)
                msg = f'Epoch:[{epoch}][{i}/{len(train_loader)}]\t'\
                        f'Loss_trip: {train_sim_loss.val:.3f}({train_sim_loss.avg:.3f})\t'\
                        f'Memory {gpu_memory_usage:.1f}'
                print(msg)
                writer = writer_dict['writer']
                global_steps = writer_dict['train_global_steps']
                writer.add_scalar('train_loss_trip', train_sim_loss.avg, global_steps)
                writer_dict['train_global_steps'] = global_steps + 1

        # store the first model
        if epoch ==  0:
            model_name =os.path.join(config.OUTPUT_DIR,
                                          f'epoch{epoch}_state.pth.tar')
            print('saving current model state to {}'.format(
                model_name))
            torch.save(model.module.state_dict(), model_name)
        # The eval part
        model.eval()
        for i, batch in enumerate(test_loader):
            q_images, g_images = batch
            if q_images.shape[0] == 1:
                continue # cannot do the triplet loss
            q_images = q_images.to(device)
            g_images = g_images.to(device)

            q_features = model(q_images)
            g_features = model(g_images)

            # calculate the loss as triplet
            trip_loss = trip_class_loss(q_features,g_features)
            test_sim_loss.update(trip_loss.item())

            if i % config.PRINT_FREQ == 0:
                gpu_memory_usage = torch.cuda.memory_allocated(0)
                msg = f'Test:[{epoch}][{i}/{len(test_loader)}]\t'\
                        f'Loss_trip: {test_sim_loss.val:.3f}({test_sim_loss.avg:.3f})\t'\
                        f'Memory {gpu_memory_usage:.1f}'
                print(msg)
                writer = writer_dict['writer']
                global_steps = writer_dict['valid_global_steps']
                writer.add_scalar('test_loss_trip', test_sim_loss.avg, global_steps)
                writer_dict['test_global_steps'] = global_steps + 1

        test_loss = test_sim_loss.avg
        # compare the loss
        if test_loss < least_test_loss:
            least_test_loss = test_loss
            best_model = True
        else:
            best_model = False

        save_checkpoint({
            'epoch': epoch + 1,
            'state_dict': model.module.state_dict(),
            'loss': test_loss,
            'optimizer': optimizer.state_dict(),
        }, best_model, config.OUTPUT_DIR)

    final_model_state_file = os.path.join(config.OUTPUT_DIR,
                                          'final_state.pth.tar')
    torch.save(model.module.state_dict(), final_model_state_file)

class AverageMeter(object):
    """Computes and stores the average and current value"""

    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

if __name__ == '__main__':
    main()


from mycroft import MycroftSkill, intent_file_handler
import RPi.GPIO as GPIO
import time

class RosGpioPins(MycroftSkill):
    def __init__(self):
        MycroftSkill.__init__(self)

    @intent_file_handler('pins.gpio.ros.intent')
    def handle_pins_gpio_ros(self, message):
        self.speak_dialog('pins.gpio.ros')
        GPIO.setmode(GPIO.BOARD)
        green=11
        GPIO.setup(green,GPIO.OUT)
        GPIO.output(green,True)
        time.sleep(1)
        GPIO.output(green,False)
        time.sleep(1)
        GPIO.output(green,True)
        time.sleep(1)
        GPIO.output(green,False)
        GPIO.cleanup()

def create_skill():
    return RosGpioPins()


#calss header
class _CALVARY():
	def __init__(self,):
		self.name = "CALVARY"
		self.definitions = [u'the hill outside the city of Jerusalem where Jesus Christ was crucified (= killed on a cross)', u'a sculpture or painting representing the crucifixion of Jesus Christ: ']

		self.parents = []
		self.childen = []
		self.properties = []
		self.jsondata = {}

		self.specie = 'nouns'

	def run(self, obj1 = [], obj2 = []):
		return self.jsondata


import cv2
import os
import numpy as np

def read_images_color(file_list):
    ims = []
    for file in file_list:
        im = cv2.imread(file)
        im = np.array([im])
        im = np.swapaxes(im, 0, 1)
        im = np.swapaxes(im, 1, 2)
        ims.append(im)
    return np.array(ims, np.float32)

def read_images_gray(file_list):
    ims = []
    for file in file_list:
        im = cv2.imread(file)
        im = np.array([cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)])
        im = np.swapaxes(im, 0, 1)
        im = np.swapaxes(im, 1, 2)
        ims.append(im)
    return np.array(ims, np.float32)

def get_file_list(root):
    file_list = []
    for file in os.listdir(root):
        file_list.append(os.path.join(root, file))
    return file_list

def save_image(files, root):
    if not os.path.exists(root):
        os.mkdir(root)

    m = np.min(files)
    l = np.max(files) - m
    files = (files - m) / l

    i = 0
    for im in files:
        cv2.imwrite(os.path.join(root, str(i) + '.png'), im)
        i += 1


from django.urls import path
from . import views
urlpatterns = [
    path('', views.index, name="index"),
    path('subjects', views.subjects, name='subjects'),
    path('subjects/<int:topics_id>', views.topics, name="topics"),
    path('subjects/ans/<int:topic_id>', views.topic, name="topic"),
    path('new_question/<subject_id>', views.new_topic, name="new_topic"),
    path('new_answer/<topic_id>', views.new_entry, name="new_entry"),
    path('edit_answer/<edit_id>', views.edit_entry, name="edit_entry"),
    path('delete_answer/<edit_id>', views.delete_entry, name="delete_entry"),
]


#!/usr/bin/env python
# encoding=utf-8 (pep 0263)

""" methods2 """

def sum_numbers(n1, n2):
    """
    Get sum of two numbers
    :param n1:
    :param n2:
    :return:
    """
    return n1 + n2

sum1 = sum_numbers(2, 8)
sum2 = sum_numbers(1, 2)
string1 = sum_numbers("a", "b"
                      )
print(sum1)
print(sum2)
print(string1)

def isLager(beer):
    list1 = [ 'Pilsner', 'Radegast', 'Dalesice' ]
    if beer in list1:
        return True
    else:
        return False

a = isLager("Pilsner")
print(a)

a = isLager("Argus")
print(a)


from utils.database_connection_utils import dbc_Utils

class sql:

    @classmethod
    def select_by_id(cls,table):
        connect = dbc_Utils.get_connect()
        cursor = dbc_Utils.get_cursor(connect)

        sql = "SELECT id FROM %s" % table
        cursor.execute(sql)

        return cursor.fetchone[0]

    @classmethod
    def delete_by_id(cls,table,id):
        connect = dbc_Utils.get_connect()
        cursor = dbc_Utils.get_cursor(connect)

        sql = "DELETE FROM %s WHERE id = %s" % (table,id)
        cursor.execute(sql)
        connect.commit()

        return cursor.rowcount


"""Common settings and globals."""

import sys
import os
from os.path import abspath, basename, dirname, join, normpath

# ########## PATH CONFIGURATION

# Absolute filesystem path to this Django project directory.
DJANGO_ROOT = dirname(dirname(abspath(__file__)))

# Site name.
SITE_NAME = basename(DJANGO_ROOT)

# Absolute filesystem path to the top-level project folder.
SITE_ROOT = dirname(DJANGO_ROOT)

# Absolute filesystem path to the secret file which holds this project's
# SECRET_KEY. Will be auto-generated the first time this file is interpreted.

SECRET_DIR = normpath(join(SITE_ROOT, 'deploy'))

if not os.path.isdir(SECRET_DIR):
    os.makedirs(SECRET_DIR)

SECRET_FILE = normpath(join(SECRET_DIR, 'SECRET_FILE'))

# Add all necessary filesystem paths to our system path so that we can use
# python import statements.
sys.path.append(SITE_ROOT)
sys.path.append(normpath(join(SITE_ROOT, 'apps')))
sys.path.append(normpath(join(SITE_ROOT, 'libs')))

# ########## END PATH CONFIGURATION

# ########## DEBUG CONFIGURATION

# Disable debugging by default.
DEBUG = False
TEMPLATE_DEBUG = DEBUG

# ########## END DEBUG CONFIGURATION

# ########## TEST RUNNER CONFIGURATION

TEST_RUNNER = 'django.test.runner.DiscoverRunner'

# ########## END TEST RUNNER CONFIGURATION

# ########## MANAGER CONFIGURATION

# Admin and managers for this project. These people receive private site
# alerts.
ADMINS = (
    ("Matthew Tiger", "matthew.c.tiger@gmail.com"),
)

MANAGERS = ADMINS

# ########## END MANAGER CONFIGURATION

# ########## GENERAL CONFIGURATION

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name although not all
# choices may be available on all operating systems. On Unix systems, a value
# of None will cause Django to use the same timezone as the operating system.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = 'America/Detroit'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html.
LANGUAGE_CODE = 'en-us'

# The ID, as an integer, of the current site in the django_site database table.
# This is used so that application data can hook into specific site(s) and a
# single database can manage content for multiple sites.
SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = False

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = False

# Sets the date format to ISO standards.
DATE_FORMAT = 'Y-m-d'
DATETIME_FORMAT = 'Y-m-d H:m:s'

# ########## END GENERAL CONFIGURATION

# ########## MEDIA CONFIGURATION

# Absolute filesystem path to the directory that will hold user-uploaded files.
MEDIA_ROOT = normpath(join(SITE_ROOT, 'media'))

# URL that handles the media served from MEDIA_ROOT.
MEDIA_URL = '/media/'

# ########## END MEDIA CONFIGURATION

# ########## STATIC FILE CONFIGURATION

# Absolute path to the directory static files should be collected to. Don't put
# anything in this directory yourself; store your static files in apps' static/
# subdirectories and in STATICFILES_DIRS.
STATIC_ROOT = normpath(join(SITE_ROOT, 'static'))

# URL prefix for static files.
STATIC_URL = '/static/'

# URL prefix for admin static files -- CSS, JavaScript and images.
ADMIN_MEDIA_PREFIX = '/static/admin/'

# Additional locations of static files.
STATICFILES_DIRS = (
    os.path.join(dirname(DJANGO_ROOT), 'templates', 'assets'),
)

# List of finder classes that know how to find static files in various
# locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder'
)

# ########## END STATIC FILE CONFIGURATION

# ########## TEMPLATE CONFIGURATION

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader'
)

# Directories to search when loading templates.
TEMPLATE_DIRS = (
    normpath(join(SITE_ROOT, 'templates')),
)

TEMPLATE_CONTEXT_PROCESSORS = (
    "django.contrib.auth.context_processors.auth",
    "django.core.context_processors.debug",
    "django.core.context_processors.i18n",
    "django.core.context_processors.media",
    "django.core.context_processors.static",
    "django.core.context_processors.tz",
    "django.contrib.messages.context_processors.messages")

# ########## END TEMPLATE CONFIGURATION

# ########## MIDDLEWARE CONFIGURATION

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
)

# ########## END MIDDLEWARE CONFIGURATION

# ########## APP CONFIGURATION

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    # Admin panel and documentation.
    'django.contrib.admin',
    'django.contrib.admindocs',

    # User-defined apps
    'rsvp'
)

# ########## END APP CONFIGURATION

# ########## URL CONFIGURATION

ROOT_URLCONF = '{}.urls'.format(SITE_NAME)

# ########## END URL CONFIGURATION

# ########## KEY CONFIGURATION

# Try to load the SECRET_KEY from our SECRET_FILE. If that fails, then generate
# a random SECRET_KEY and save it into our SECRET_FILE for future loading. If
# everything fails, then just raise an exception.
characters = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)'
try:
    with open(SECRET_FILE, 'r') as f:
        SECRET_KEY = f.read().strip()
except IOError:
    try:
        import random
        SECRET_KEY = ''.join([random.SystemRandom().choice(characters)
                              for i in range(50)])
        with open(SECRET_FILE, 'w') as f:
            f.write(SECRET_KEY)
    except IOError:
        message = 'Cannot open file `{}` for writing.'
        raise Exception(message.format(SECRET_FILE))

# ########## END KEY CONFIGURATION


from pyfiglet import figlet_format
from termcolor import colored
import requests
from random import choice

URL = "https://icanhazdadjoke.com/search"

def print_heading(heading):
    heading = colored(figlet_format(heading), color="magenta")
    print(heading)

def find_joke(term):
    response = requests.get(
        URL,
        headers={
            "Accept": "application/json"
        },
        params={
            "term": term
        }
    )
    results = response.json()["results"]
    print_joke(results, term)

def print_joke(results, term):
    if not results:
        return print(f"Sorry, I don't have any jokes about {term}! Please try again.")
    else:
        print("I've got one joke about fruit. Here it is:")
        print(choice(results)["joke"])

print_heading("Dad Joke 3000")
topic = input("Let me tell you a joke! Give me a topic: ")
find_joke(topic)


# -*- coding: utf-8 -*-

"""
 * Copyright(C) Anyuntec 2017.
 *
 * 攻击的调度类
 *
 * @author margin 2017/02/20.
 *
 * @version V1.00.
 *
 * 更新履历： V1.00 2017/02/20 margin 创建.
 *
 """
from AttackObject import AttackObject
from SSHAttack import SSHAttack
import sys

class Attacker():
    """
    攻击调度类
    """
    attackObject = AttackObject()
    def __init__(self,attackObject):
        self.attackObject = attackObject
        reload(sys)
        sys.setdefaultencoding('utf8')

    def attack(self,attack_data):
        """
        attack_data的数据格式为前台传递的格式
        :param attack_data: {"ip":[80,3306],"ip2":[22]}
        :return:
        """
        for ip,ports in attack_data.items():
            self.attackObject.ip = ip
            if ports:
                for port in ports:
                    if port == "22" or port == 22:
                        sshAttacker = SSHAttack(self.attackObject)
                        sshAttacker.attack()

if __name__ == "__main__":
    attackObject = AttackObject()
    attackObject.threads = 5
    attackObject.timeout = 32
    attackObject.usernames = "username.txt"
    attackObject.passwords = "password.txt"

    attacker = Attacker(attackObject)
    attacker.attack({"192.168.1.223":[80,3306],"192.168.1.222":[22]})


from confluent_kafka import Producer
from typing import Sequence

from snuba import settings, util
from snuba.consumer import ConsumerWorker
from snuba.consumers.snapshot_worker import SnapshotAwareWorker
from snuba.datasets.factory import enforce_table_writer, get_dataset
from snuba.snapshots import SnapshotId
from snuba.stateful_consumer.control_protocol import TransactionData
from snuba.utils.streams.batching import BatchingConsumer
from snuba.utils.streams.kafka import KafkaConsumer, KafkaConsumerWithCommitLog, KafkaMessage, TransportError, build_kafka_consumer_configuration

class ConsumerBuilder:
    """
    Simplifies the initialization of a batching consumer by merging
    parameters that generally come from the command line with defaults
    that come from the dataset class and defaults that come from the
    settings file.
    """

    def __init__(
        self,
        dataset_name: str,
        raw_topic: str,
        replacements_topic: str,
        max_batch_size: int,
        max_batch_time_ms: int,
        bootstrap_servers: Sequence[str],
        group_id: str,
        commit_log_topic: str,
        auto_offset_reset: str,
        queued_max_messages_kbytes: int,
        queued_min_messages: int,
        dogstatsd_host: str,
        dogstatsd_port: int
    ) -> None:
        self.dataset = get_dataset(dataset_name)
        self.dataset_name = dataset_name
        if not bootstrap_servers:
            self.bootstrap_servers = settings.DEFAULT_DATASET_BROKERS.get(
                dataset_name,
                settings.DEFAULT_BROKERS,
            )
        else:
            self.bootstrap_servers = bootstrap_servers

        stream_loader = enforce_table_writer(self.dataset).get_stream_loader()
        self.raw_topic = raw_topic or stream_loader.get_default_topic_spec().topic_name
        default_replacement_topic_name = stream_loader.get_replacement_topic_spec().topic_name \
            if stream_loader.get_replacement_topic_spec() \
            else None
        self.replacements_topic = replacements_topic or default_replacement_topic_name
        default_commit_log_topic_name = stream_loader.get_commit_log_topic_spec().topic_name \
            if stream_loader.get_commit_log_topic_spec() \
            else None
        self.commit_log_topic = commit_log_topic or default_commit_log_topic_name

        # XXX: This can result in a producer being built in cases where it's
        # not actually required.
        self.producer = Producer({
            'bootstrap.servers': ','.join(self.bootstrap_servers),
            'partitioner': 'consistent',
            'message.max.bytes': 50000000,  # 50MB, default is 1MB
        })

        self.metrics = util.create_metrics(
            dogstatsd_host, dogstatsd_port, 'snuba.consumer',
            tags={
                "group": group_id,
                "dataset": self.dataset_name,
            }
        )

        self.max_batch_size = max_batch_size
        self.max_batch_time_ms = max_batch_time_ms
        self.group_id = group_id
        self.auto_offset_reset = auto_offset_reset
        self.queued_max_messages_kbytes = queued_max_messages_kbytes
        self.queued_min_messages = queued_min_messages

    def __build_consumer(self, worker: ConsumerWorker) -> BatchingConsumer:
        configuration = build_kafka_consumer_configuration(
            bootstrap_servers=self.bootstrap_servers,
            group_id=self.group_id,
            auto_offset_reset=self.auto_offset_reset,
            queued_max_messages_kbytes=self.queued_max_messages_kbytes,
            queued_min_messages=self.queued_min_messages,
        )

        if self.commit_log_topic is None:
            consumer = KafkaConsumer(configuration)
        else:
            consumer = KafkaConsumerWithCommitLog(
                configuration,
                self.producer,
                self.commit_log_topic,
            )

        return BatchingConsumer(
            consumer,
            self.raw_topic,
            worker=worker,
            max_batch_size=self.max_batch_size,
            max_batch_time=self.max_batch_time_ms,
            metrics=self.metrics,
            recoverable_errors=[TransportError],
        )

    def build_base_consumer(self) -> BatchingConsumer:
        """
        Builds the consumer with a ConsumerWorker.
        """
        return self.__build_consumer(
            ConsumerWorker(
                self.dataset,
                producer=self.producer,
                replacements_topic=self.replacements_topic,
                metrics=self.metrics
            )
        )

    def build_snapshot_aware_consumer(
        self,
        snapshot_id: SnapshotId,
        transaction_data: TransactionData,
    ) -> BatchingConsumer:
        """
        Builds the consumer with a ConsumerWorker able to handle snapshots.
        """
        worker = SnapshotAwareWorker(
            dataset=self.dataset,
            producer=self.producer,
            snapshot_id=snapshot_id,
            transaction_data=transaction_data,
            metrics=self.metrics,
            replacements_topic=self.replacements_topic,
        )
        return self.__build_consumer(worker)


import math
import os
import warnings
from dataclasses import dataclass
from typing import Optional, Tuple

import torch
import torch.utils.checkpoint
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss
from transformers import BertPreTrainedModel, BertModel
from transformers.modeling_outputs import SequenceClassifierOutput

class BertForSharedTaskBaseline(BertPreTrainedModel):

    _keys_to_ignore_on_load_unexpected = [r"pooler"]

    def __init__(self, config):
        super().__init__(config)
        self.num_labels = config.num_labels

        self.bert = BertModel(config)
        self.dropout = nn.Dropout(config.hidden_dropout_prob)
        self.classifier = nn.Linear(config.hidden_size, config.num_labels)

        self.init_weights()

    def forward(
        self,
        input_ids=None,
        target_positions=None,
        attention_mask=None,
        token_type_ids=None,
        position_ids=None,
        head_mask=None,
        inputs_embeds=None,
        labels=None,
        output_attentions=None,
        output_hidden_states=None,
        return_dict=None,
    ):
        r"""
        labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
            Labels for computing the token classification loss. Indices should be in ``[0, ..., config.num_labels -
            1]``.
        """
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        outputs = self.bert(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        pooled_output = outputs[0][torch.arange(outputs[0].size(0)), target_positions]
        pooled_output = self.dropout(pooled_output)
        logits = self.classifier(pooled_output)

        loss = None
        if labels is not None:
            if self.num_labels == 1:
                #  We are doing regression
                loss_fct = MSELoss()
                loss = loss_fct(logits.view(-1), labels.view(-1))
            else:
                loss_fct = CrossEntropyLoss()
                loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))

        if not return_dict:
            output = (logits,) + outputs[2:]
            return ((loss,) + output) if loss is not None else output

        return SequenceClassifierOutput(
            loss=loss,
            logits=logits,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
        )


from sqlalchemy.exc import InvalidRequestError, IntegrityError
from config.DBindex import db_session
from models.order import ClientOrders
from models.mission import MissionClientOrderRelate
from pkg.logger import get_logger as log
from datetime import datetime
from pkg.checkDictMatch import checkDictKeyMatchArray

modelKey = [
    "satellite_name" ,
    "weight_kg" ,
    "purpose" ,
    "request_by" ,
    "eta_height_km" ,
    "arrival_date" ,
    "inclination" ,
    "budget_billion",
    "launch_day",
    "status"
]

def FindAll():
    try:
        query = ClientOrders.query.all()
        dataDict = []
        for data in query:
            data.__dict__.pop("_sa_instance_state")
            dataDict.append(data.__dict__)
        return dataDict, 200
    except :
        log().error("clientOrder controller findall")
        return None, 404

def FindOne(cond):
    try:
        """if "id" not in cond:
            return None, 400
        orderID = cond.pop("id")

        querydict, isMatch = checkDictKeyMatchArray(modelKey, cond)
        if not isMatch:
            return None, 400"""
        query = ClientOrders.query.filter_by(id=cond)

        if query.one_or_none() is not None:
            q = query.one_or_none()
            q.__dict__.pop("_sa_instance_state")
            return q.__dict__, 200
        else:
            return None, 404
    except InvalidRequestError:
        log().error("InvalidRequestError")
        return None, 400

def Create(cond):
    querydict = {}
    querydict, isMatch = checkDictKeyMatchArray(modelKey, cond)
    if not isMatch:
        return None, 400

    createClientOrder = ClientOrders(**querydict)

    try:
        db_session.add(createClientOrder)
        db_session.commit()
        return querydict, 200
    except InvalidRequestError:
        log().error("Unable to create data")
        return None, 400
    except IntegrityError:
        log().error("Foreign key not found")
        return None, 400

def Patch(content):
    try:
        if not content['id']:
            return None, 400
        query = ClientOrders.query.filter_by(id=content.pop("id")).one_or_none()
        if query is not None:
            querydict = {}
            querydict, isMatch = checkDictKeyMatchArray(modelKey, content)
            if not isMatch:
                return None, 400
            for key in querydict:
                setattr(query, key, querydict[key])
            db_session.commit()
            return querydict, 200
        else:
            return None, 404
    except InvalidRequestError:
        log().error("Unable to patch data")
        return None, 400

def Delete(id):
    try:
        toDel = ClientOrders.query.filter_by(id=id).first()
        if toDel is not None:
            delRelate = MissionClientOrderRelate.query.filter_by(clientOrder_id=id)
            if delRelate.one_or_none() is not None:
                for data in delRelate.all():
                    db_session.delete(data)
            db_session.commit()

            db_session.delete(toDel)
            db_session.commit()
            return 200
        else:
            return 400
    except InvalidRequestError:
        log().error("Unable to delete clientOrder data")
        return 400


#!/usr/bin/env python3
import sys
import os

def print_args(args):
    print(args[-1] + ' '+  '_'.join(args[:-1]))

def main(argv):
    path = "%STANDARD/PATDB"
    # If directory does not exist, print random stuff
    if not os.path.exists(path):
        print("error error")
        return
    args = []
    with open(path, 'r') as infile:
        for line in infile:
            line = line.strip()
            if not line or line.startswith('#'):
                continue
            if line.startswith('$'):
                print_args(args)
                args = []
            else:
                args.append(line)

if __name__ == '__main__':
    main(sys.argv)


from xai.brain.wordbase.nouns._bathhouse import _BATHHOUSE

#calss header
class _BATHHOUSES(_BATHHOUSE, ):
	def __init__(self,):
		_BATHHOUSE.__init__(self)
		self.name = "BATHHOUSES"
		self.specie = 'nouns'
		self.basic = "bathhouse"
		self.jsondata = {}


from Crypto.Util.number import inverse
from random import randint

class elgamal:
    def __init__(self,key):
        self.p , self.q , self.g , self.y = key[:4]
        if len(key) == 5:
            self.x = key[4]
        else:
            self.x = None

    def encrypt(self,m):
        r = randint(1 , self.q)
        y1 = pow(self.g , r , self.p)
        y2 = m * pow(self.y , r , self.p) % self.p
        return y1 , y2

    def decrypt(self,c):
        if self.x != None:
           y1 , y2 = c
           m = y2 * inverse(pow(y1 , self.x , self.p), self.p) % self.p
           return m
        else:
            print('can\'t decrypt without private key')
            return -1


from maya import cmds

import pyblish.api
import colorbleed.api
import colorbleed.maya.action
import colorbleed.maya.lib as lib

class ValidateRenderNoDefaultCameras(pyblish.api.InstancePlugin):
    """Ensure no default (startup) cameras are to be rendered."""

    order = colorbleed.api.ValidateContentsOrder
    hosts = ['maya']
    families = ['colorbleed.renderlayer']
    label = "No Default Cameras Renderable"
    actions = [colorbleed.maya.action.SelectInvalidAction]

    @staticmethod
    def get_invalid(instance):

        layer = instance.data["setMembers"]

        # Collect default cameras
        cameras = cmds.ls(type='camera', long=True)
        defaults = [cam for cam in cameras if
                    cmds.camera(cam, query=True, startupCamera=True)]

        invalid = []
        with lib.renderlayer(layer):
            for cam in defaults:
                if cmds.getAttr(cam + ".renderable"):
                    invalid.append(cam)

        return invalid

    def process(self, instance):
        """Process all the cameras in the instance"""
        invalid = self.get_invalid(instance)
        if invalid:
            raise RuntimeError("Renderable default cameras "
                               "found: {0}".format(invalid))


import concurrent.futures
import json

import requests
from tqdm import tqdm

raw_text = "Slicing can be best visualized by considering the index to be between the elements as shown below. So if we want to access a range, we need two indices that will slice that portion from the list."
data = raw_text.split()
DATA = [w.upper() for w in data]

# Retrieve a single page and report the URL and contents
def get_lower(word):
    with requests.get('http://localhost:5000/api/lowercase/{}'.format(word)) as response:
        json_data = json.loads(response.text)
        return json_data['task']

if __name__ == '__main__':
    result = []
    data_list  = tqdm(DATA)
    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
        future_to_url = {executor.submit(get_lower, w): w for w in DATA}
        for future in concurrent.futures.as_completed(future_to_url):
            url = future_to_url[future]
            try:
                data = future.result()
            except Exception as exc:
                print('%r generated an exception: %s' % (url, exc))
            else:
                print('%r page is %d bytes' % (url, len(data)))


import cv2
import tensorflow as tf
import numpy as np
sess = tf.Session('', tf.Graph())
cam = cv2.VideoCapture(0)
with sess.graph.as_default():
	# Read meta graph and checkpoint to restore session
	saver = tf.train.import_meta_graph("cnn_files\model.ckpt-10012.meta")
	saver.restore(sess,"cnn_files\model.ckpt-10012")
	while True:
		_, frame = cam.read()
		image = cv2.resize(frame, (32, 32), interpolation=cv2.INTER_CUBIC)
		image = tf.expand_dims(image, axis=0)
		image = tf.cast(image, tf.float32, name="image")
		image = sess.run(image)
		# Start the queue runners. If they are not started the program will hang
		coord = tf.train.Coordinator()
		threads =[]
		for qr in sess.graph.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
			threads.extend(qr.create_threads(sess, coord=coord, daemon=True,
											 start=True))

		# In the graph created above, feed "is_training" and "imgs" placeholders
		# Feeding them will disconnect the path from queue runners to the graph
		# and enable a path from the placeholder instead. The "img" placeholder will be
		# fed with the image that was read above.
		logits = sess.run('softmax_linear/softmax_linear:0',
						  feed_dict={'is_training:0': False, 'imgs:0':image})
		#Print classification results
		print(logits)


# -*- coding: utf-8 -*-
"""
.. module:: bbf-multi-localrepos.hooks.git_update
    :synopsis: hook to update git repos

copyright 2016 Brown Bag Films
"""
from __future__ import print_function

__all__ = ['GitUpdateHook']

import os
import urlparse
import re
import time

import sgtk

HookBaseClass = sgtk.get_hook_baseclass()

class GitUpdateHook(HookBaseClass):
    def execute(self, repo, progress):

        progress.update_output.emit('Updating local {repo} repository'.format(repo=repo['name']))

        local_repo = repo.get("local_url")
        check_git_lock(local_repo)

        process = self.parent.update_subprocess(repo['local_url'], branch=repo['branch'])

        if process is None:
            error_msg = "An Error has occurred trying to update %s\n" % repo['name']
            # self.parent.log_info(error_msg)
            progress.update_output.emit(error_msg)
            return False

        percentage_re = re.compile('(?P<percentage>\d+)%')
        t = 0
        if process.waitForStarted(60000):
            while process.state() == process.Running:
                if t >= 10:
                    self.parent.clean_process(process)
                    break
                t += 1
                if process.waitForReadyRead(60000):
                    available_output = str(process.readAllStandardOutput()).replace("\r", "")
                    for line in available_output.split("\n"):
                        progress.update_output.emit(line)
                        m = percentage_re.search(line)
                        if m is not None:
                            progress.update_percentage.emit(float(m.group("percentage")))

        process.close()
        progress.update_output.emit('{repo} up to date'.format(repo=repo['name']))

        return True

def check_git_lock(local_repo_path):
    """
    Checks '.git' folder for 'index.lock' and delete it if exists and is longer than 24 hours old.

    :param local_repo_path: str
    """
    cut_off_time = 86400.0  # 24 hours in seconds

    if local_repo_path is not None:
        p = urlparse.urlparse(local_repo_path)
        local_repo_path = os.path.abspath(os.path.join(p.netloc, p.path))

        git_lock_file = os.path.join(local_repo_path, ".git", "index.lock")

        if os.path.exists(git_lock_file):
            time_modified = os.path.getmtime(git_lock_file)
            now = time.time()

            if (now - time_modified) > cut_off_time:
                try:
                    os.remove(git_lock_file)
                except IOError as e:
                    pass


# Project Crime - Dhivya Sivaramakrishnan, Mangesh Bhangare

from pyspark import SparkConf, SparkContext
from pyspark.sql import SQLContext, DataFrame, Row
from pyspark.mllib.clustering import KMeans, KMeansModel
from numpy import array
from math import sqrt

import sys

conf = SparkConf().setAppName('K-Means test')
sc = SparkContext(conf=conf)
assert sc.version >= '1.5.1'
sqlContext = SQLContext(sc)

# Read the input parquet
input_crime = sys.argv[1]

# Read the parquet data and convert to RDD
parquet_crime = sqlContext.read.parquet(input_crime)
parquet_crime.registerTempTable("crime_table")
crime_table = sqlContext.sql("SELECT * FROM crime_table")
crime_rdd = crime_table.map(lambda line: str(line.Year) + "," + str(line.Latitude) + ","
                                       + str(line.Longitude) + "," + str(line.Crime_Frequency))

# K-means does multiple runs to find the optimal cluster center, so cache the input to K-means
cluster_input = crime_rdd.map(lambda line: array([float(x) for x in line.split(',')])).cache()

# Perform K-means clustering
clusters = KMeans.train(cluster_input, 20, maxIterations=5,
        runs=5, initializationMode="random")

# Compute root mean squared error and change cluster centers
def squared_error(point):
    center = clusters.centers[clusters.predict(point)]
    return sqrt(sum([x**2 for x in (point - center)]))

error = cluster_input.map(lambda point: squared_error(point)).reduce(lambda x, y: x + y)
print("Squared error for a cluster = " + str(error))

clusters.save(sc, "myModel_crime/crime")
sameModel = KMeansModel.load(sc, "myModel_crime/crime")


from fastText import load_model
import torch
import numpy as np
import sentencepiece as spm

sp = spm.SentencePieceProcessor()
dirname = "/home/pragma/development/deeplearning/"

def main():
	words, embedding = createEmbedding()
	mytext = "Cengiz Han’ın büyük amacası Kutua, bu hakarete Çin üzerine ve Tatarlara bir dizi saldırı düzenleyerek cevap verdi ve bu akınlar sonunda “Moğol Herkülü” unvanını kazandı."
	# mytext = input("Text 1: ")
	myothertext = "Makarna mutfaklarımızda çoğunlukla olan, severek tüketilen bir lezzettir. Haşlanmış ve soslu makarna tariflerine artık sıkılanlar için kolay ve en lezzetli şekliyle beşamelli fırında makarna tarifimi sizlerle paylaşıyorum."
	# myothertext = input("Text 2: ")
	print("Text 1: " + mytext)
	print("Text 1: " + myothertext)
	print("Cos sim (mean): " + str(findSim(mytext, myothertext, words, embedding, "mean")))
	print("Cos sim (sum): " + str(findSim(mytext, myothertext, words, embedding, "sum")))

def findSim(text1, text2, words, embedding, method):
	text1v = findSentenceVec(text1, words, embedding, method)
	text2v = findSentenceVec(text2, words, embedding, method)
	return torch.nn.functional.cosine_similarity(text1v, text2v)

def findSentenceVec(text, words, embedding, method):
	sp.Load(dirname + "sentencepiecepython/sentences.model")
	text = text.lower()
	text = sp.EncodeAsPieces(text.encode('utf-8'))
	ids = [words[word] for word in text]
	vs = embedding(torch.LongTensor(ids))
	if method == "sum":
		return torch.sum(vs, dim = 0, keepdim = True)
	elif method == "mean":
		return torch.mean(vs, dim = 0, keepdim = True)
	return None

def createEmbedding():
	f = load_model(dirname + "fasttextresult/result-sp-10e.bin")
	words = f.get_words()
	print(str(len(words)) + " " + str(f.get_dimension()))

	mywords = {}
	mylist = []

	i = 0
	for w in words:
		mywords[w] = i
		mylist.append(f.get_word_vector(w))
		i = i + 1
	matrix = np.array(mylist)
	tensor = torch.from_numpy(matrix)
	embedding = torch.nn.Embedding.from_pretrained(tensor, freeze = True)
	return mywords, embedding

if __name__ == "__main__":
	main()