import datetime
import pytest

from model_bakery import baker
from rest_framework import status
from uuid import UUID
from unittest.mock import Mock

from usaspending_api.common.exceptions import InvalidParameterException
from usaspending_api.recipient.v2.views import recipients

# Getting relative dates as the 'latest'/default argument returns results relative to when it gets called
from usaspending_api.search.tests.data.utilities import setup_elasticsearch_test

TODAY = datetime.datetime.now()
INSIDE_OF_LATEST = TODAY - datetime.timedelta(365 - 2)
FISCAL_INSIDE_OF_LATEST = INSIDE_OF_LATEST + datetime.timedelta(days=30 * 3)

TEST_REF_COUNTRY_CODE = {
    "PARENT COUNTRY CODE": {"country_code": "PARENT COUNTRY CODE", "country_name": "PARENT COUNTRY NAME"},
    "CHILD COUNTRY CODE": {"country_code": "CHILD COUNTRY CODE", "country_name": "CHILD COUNTRY NAME"},
}
MAP_DUNS_TO_CONTRACT = {
    "address_line_1": "address_line1",
    "address_line_2": "address_line2",
    "city": "city_name",
    "congressional_district": "congressional_code",
    "state": "state_code",
    "zip5": "zip",
}

TEST_DUNS = {
    "000000001": {
        "awardee_or_recipient_uniqu": "000000001",
        "legal_business_name": "PARENT RECIPIENT",
        "address_line_1": "PARENT ADDRESS LINE 1",
        "address_line_2": "PARENT ADDRESS LINE 2",
        "city": "PARENT CITY",
        "congressional_district": "PARENT CONGRESSIONAL DISTRICT",
        "country_code": "PARENT COUNTRY CODE",
        "state": "PARENT STATE",
        "zip": "PARENT ZIP",
        "zip4": "PARENT ZIP4",
        "business_types_codes": ["2X"],
        "uei": "AAAAAAAAAAAA",
    },
    "000000002": {
        "awardee_or_recipient_uniqu": "000000002",
        "legal_business_name": "CHILD RECIPIENT",
        "address_line_1": "CHILD ADDRESS LINE 1",
        "address_line_2": "CHILD ADDRESS LINE 2",
        "city": "CHILD CITY",
        "congressional_district": "CHILD CONGRESSIONAL DISTRICT",
        "country_code": "CHILD COUNTRY CODE",
        "state": "CHILD STATE",
        "zip": "CHILD ZIP",
        "zip4": "CHILD ZIP4",
        "business_types_codes": ["A8"],
    },
}
TEST_RECIPIENT_LOCATIONS = {
    "a52a7544-829b-c925-e1ba-d04d3171c09a": {
        "address_line_1": "PARENT ADDRESS LINE 1",
        "address_line_2": "PARENT ADDRESS LINE 2",
        "city": "PARENT CITY",
        "congressional_district": "PARENT CONGRESSIONAL DISTRICT",
        "country_code": "PARENT COUNTRY CODE",
        "state": "PARENT STATE",
        "zip5": "PARENT ZIP",
        "zip4": "PARENT ZIP4",
    },
    "acb93cfc-e4f8-ecd5-5ac3-fa62f115e8f5": {
        "address_line_1": "CHILD ADDRESS LINE 1",
        "address_line_2": "CHILD ADDRESS LINE 2",
        "city": "CHILD CITY",
        "congressional_district": "CHILD CONGRESSIONAL DISTRICT",
        "country_code": "CHILD COUNTRY CODE",
        "state": "CHILD STATE",
        "zip5": "CHILD ZIP",
        "zip4": "CHILD ZIP4",
    },
    "cd6ac2ed-8d2d-2c2e-4934-710eb82100f3": {
        "address_line_1": "OTHER ADDRESS LINE 1",
        "address_line_2": "OTHER ADDRESS LINE 2",
        "city": "OTHER CITY",
        "congressional_district": "OTHER CONGRESSIONAL DISTRICT",
        "country_code": "OTHER COUNTRY CODE",
        "state": "OTHER STATE",
        "zip5": "OTHER ZIP",
        "zip4": "OTHER ZIP4",
    },
    "6dffe44a-554c-26b4-b7ef-44db50083732": {
        "address_line_1": None,
        "address_line_2": None,
        "city": None,
        "congressional_district": None,
        "country_code": None,
        "state": None,
        "zip5": None,
        "zip4": None,
    },
}
TEST_RECIPIENT_LOOKUPS = {
    "a52a7544-829b-c925-e1ba-d04d3171c09a": {
        "recipient_hash": "a52a7544-829b-c925-e1ba-d04d3171c09a",
        "duns": "000000001",
        "uei": "AAAAAAAAAAAA",
        "legal_business_name": "PARENT RECIPIENT",
    },
    "acb93cfc-e4f8-ecd5-5ac3-fa62f115e8f5": {
        "recipient_hash": "acb93cfc-e4f8-ecd5-5ac3-fa62f115e8f5",
        "duns": "000000002",
        "uei": "BBBBBBBBBBBB",
        "legal_business_name": "CHILD RECIPIENT",
    },
    "cd6ac2ed-8d2d-2c2e-4934-710eb82100f3": {
        "recipient_hash": "cd6ac2ed-8d2d-2c2e-4934-710eb82100f3",
        "duns": None,
        "uei": None,
        "legal_business_name": "OTHER RECIPIENT",
    },
    "6dffe44a-554c-26b4-b7ef-44db50083732": {
        "recipient_hash": "6dffe44a-554c-26b4-b7ef-44db50083732",
        "duns": None,
        "uei": None,
        "legal_business_name": "MULTIPLE RECIPIENTS",
    },
}
for hash, recipient in TEST_RECIPIENT_LOOKUPS.items():
    recipient.update(TEST_RECIPIENT_LOCATIONS[hash])

TEST_RECIPIENT_PROFILES = {
    # Parent Recipient, including non-existent child duns
    "a52a7544-829b-c925-e1ba-d04d3171c09a-P": {
        "recipient_level": "P",
        "recipient_hash": "a52a7544-829b-c925-e1ba-d04d3171c09a",
        "recipient_unique_id": "000000001",
        "recipient_name": "PARENT RECIPIENT",
        "recipient_affiliations": [
            "AAAAAAAAAAAA",
            "BBBBBBBBBBBB",
            "ZZZZZZZZZZZZ",
        ],
        "uei": "AAAAAAAAAAAA",
    },
    # Child Recipient 1 - lists itself as both parent and child
    "a52a7544-829b-c925-e1ba-d04d3171c09a-C": {
        "recipient_level": "C",
        "recipient_hash": "a52a7544-829b-c925-e1ba-d04d3171c09a",
        "recipient_unique_id": "000000001",
        "recipient_name": "PARENT RECIPIENT",
        "recipient_affiliations": ["AAAAAAAAAAAA"],
        "uei": "AAAAAAAAAAAA",
    },
    # Child Recipient 2 - different from parent duns
    "acb93cfc-e4f8-ecd5-5ac3-fa62f115e8f5-C": {
        "recipient_level": "C",
        "recipient_hash": "acb93cfc-e4f8-ecd5-5ac3-fa62f115e8f5",
        "recipient_unique_id": "000000002",
        "recipient_name": "CHILD RECIPIENT",
        "recipient_affiliations": ["AAAAAAAAAAAA"],
        "uei": "BBBBBBBBBBBB",
    },
    # Other Recipient
    "cd6ac2ed-8d2d-2c2e-4934-710eb82100f3-R": {
        "recipient_level": "R",
        "recipient_hash": "cd6ac2ed-8d2d-2c2e-4934-710eb82100f3",
        "recipient_unique_id": None,
        "recipient_name": "OTHER RECIPIENT",
        "recipient_affiliations": [],
        "uei": "BBBBBBBBBBBB",
    },
}

TEST_SUMMARY_TRANSACTION_NORMALIZED_FOR_FPDS = {
    "latest": {
        "action_date": INSIDE_OF_LATEST,
        "fiscal_action_date": FISCAL_INSIDE_OF_LATEST,
        "federal_action_obligation": 100,
        "generated_pragmatic_obligation": 100,
    },
    "FY2016": {
        "action_date": datetime.datetime(2015, 10, 1),
        "fiscal_action_date": datetime.datetime(2016, 1, 1),
        "federal_action_obligation": 50,
        "generated_pragmatic_obligation": 50,
    },
    "FY2008": {
        "action_date": datetime.datetime(2007, 10, 1),
        "fiscal_action_date": datetime.datetime(2008, 1, 1),
        "federal_action_obligation": 200,
        "generated_pragmatic_obligation": 200,
    },
}
TEST_SUMMARY_TRANSACTION_NORMALIZED_FOR_FABS = {
    "latest": {
        "action_date": INSIDE_OF_LATEST,
        "fiscal_action_date": FISCAL_INSIDE_OF_LATEST,
        "face_value_loan_guarantee": 1000,
        "type": "07",
        "generated_pragmatic_obligation": 0,
    },
    "FY2016": {
        "action_date": datetime.datetime(2015, 10, 1),
        "fiscal_action_date": datetime.datetime(2016, 1, 1),
        "face_value_loan_guarantee": 500,
        "type": "08",
        "generated_pragmatic_obligation": 0,
    },
    "FY2008": {
        "action_date": datetime.datetime(2007, 10, 1),
        "fiscal_action_date": datetime.datetime(2008, 1, 1),
        "face_value_loan_guarantee": 2000,
        "type": "08",
        "generated_pragmatic_obligation": 0,
    },
}
TEST_SUMMARY_TRANSACTION_RECIPIENT = {
    "latest": {
        "recipient_name": "PARENT RECIPIENT",
        "recipient_unique_id": "000000001",
        "parent_recipient_unique_id": "000000001",
        "recipient_uei": "AAAAAAAAAAAA",
        "parent_uei": "AAAAAAAAAAAA",
        "recipient_hash": "a52a7544-829b-c925-e1ba-d04d3171c09a",
        "parent_recipient_hash": "a52a7544-829b-c925-e1ba-d04d3171c09a",
        "recipient_levels": ["P", "C"],
    },
    "FY2016": {
        "recipient_name": "CHILD RECIPIENT",
        "recipient_unique_id": "000000002",
        "parent_recipient_unique_id": "000000001",
        "recipient_uei": "BBBBBBBBBBBB",
        "parent_uei": "AAAAAAAAAAAA",
        "recipient_hash": "acb93cfc-e4f8-ecd5-5ac3-fa62f115e8f5",
        "parent_recipient_hash": "a52a7544-829b-c925-e1ba-d04d3171c09a",
        "recipient_levels": ["C"],
    },
    "FY2008": {
        "recipient_name": "OTHER RECIPIENT",
        "recipient_unique_id": None,
        "parent_recipient_unique_id": None,
        "parent_uei": None,
        "recipient_hash": "cd6ac2ed-8d2d-2c2e-4934-710eb82100f3",
        "recipient_levels": ["R"],
    },
}

@pytest.mark.django_db
def create_transaction_test_data(transaction_recipient_list=None):

    if transaction_recipient_list is None:
        transaction_recipient_list = list(TEST_SUMMARY_TRANSACTION_RECIPIENT.values())

    id = 1

    for count, transaction_search in enumerate(TEST_SUMMARY_TRANSACTION_NORMALIZED_FOR_FPDS.values()):
        base_transaction_search = {
            "transaction_id": id,
            "award_id": id,
            "is_fpds": True,
            "business_categories": ["expected", "business", "cat"],
        }
        base_transaction_search.update(transaction_search)
        base_transaction_search.update(transaction_recipient_list[count])
        baker.make("search.AwardSearch", award_id=id, latest_transaction_id=id)
        baker.make("search.TransactionSearch", **base_transaction_search)

        id += 1

    for count, transaction_search in enumerate(TEST_SUMMARY_TRANSACTION_NORMALIZED_FOR_FABS.values()):
        base_transaction_search = {
            "transaction_id": id,
            "award_id": id,
            "is_fpds": False,
            "business_categories": ["expected", "business", "cat"],
        }
        base_transaction_search.update(transaction_search)
        base_transaction_search.update(transaction_recipient_list[count])
        baker.make("search.AwardSearch", award_id=id, latest_transaction_id=id)
        baker.make("search.TransactionSearch", **base_transaction_search)

        id += 1

@pytest.mark.django_db
def create_recipient_profile_test_data(*recipient_profile_list):
    for recipient_profile in recipient_profile_list:
        baker.make("recipient.RecipientProfile", **recipient_profile)

@pytest.mark.django_db
def create_recipient_lookup_test_data(*recipient_lookup_list):
    for recipient_lookup in recipient_lookup_list:
        baker.make("recipient.RecipientLookup", **recipient_lookup)

@pytest.mark.django_db
def test_validate_recipient_id_success():
    """ Testing a run of a valid recipient id """
    recipient_id = "a52a7544-829b-c925-e1ba-d04d3171c09a-P"
    baker.make("recipient.RecipientProfile", **TEST_RECIPIENT_PROFILES[recipient_id])

    expected_hash = recipient_id[:-2]
    expected_level = recipient_id[-1]
    try:
        recipient_hash, recipient_level = recipients.validate_recipient_id(recipient_id)
        assert recipient_hash == expected_hash
        assert recipient_level == expected_level
    except InvalidParameterException:
        assert False

@pytest.mark.django_db
def test_validate_recipient_id_failures():
    """ Testing a run of invalid recipient ids """
    recipient_id = "a52a7544-829b-c925-e1ba-d04d3171c09a-P"
    baker.make("recipient.RecipientProfile", **TEST_RECIPIENT_PROFILES[recipient_id])

    def call_validate_recipient_id(recipient_id):
        try:
            recipients.validate_recipient_id(recipient_id)
            return False
        except InvalidParameterException:
            return True

    # Test with no dashes
    recipient_id = "broken_recipient_id"
    assert call_validate_recipient_id(recipient_id) is True

    # Test with invalid recipient level
    recipient_id = "broken_recipient-id"
    assert call_validate_recipient_id(recipient_id) is True

    # Test with invalid hash
    recipient_id = "broken_recipient-R"
    assert call_validate_recipient_id(recipient_id) is True

    # Test with id not available
    recipient_id = "00002940-fdbe-3fc5-9252-000000-R"
    assert call_validate_recipient_id(recipient_id) is True

@pytest.mark.django_db
def test_extract_duns_uei_name_from_hash():
    """ Testing extracting name and duns from the recipient hash """
    recipient_hash = "a52a7544-829b-c925-e1ba-d04d3171c09a"
    baker.make("recipient.RecipientLookup", **TEST_RECIPIENT_LOOKUPS[recipient_hash])

    expected_duns = TEST_RECIPIENT_LOOKUPS[recipient_hash]["duns"]
    expected_uei = TEST_RECIPIENT_LOOKUPS[recipient_hash]["uei"]
    expected_name = TEST_RECIPIENT_LOOKUPS[recipient_hash]["legal_business_name"]
    duns, uei, name = recipients.extract_duns_uei_name_from_hash(recipient_hash)
    assert duns == expected_duns
    assert uei == expected_uei
    assert name == expected_name

@pytest.mark.django_db
def test_extract_parent_from_hash():
    """ Testing extracting parent duns/name from recipient hash"""
    # This one specifically has to be a child
    recipient_id = "acb93cfc-e4f8-ecd5-5ac3-fa62f115e8f5-C"
    recipient_hash = TEST_RECIPIENT_PROFILES[recipient_id]["recipient_hash"]
    baker.make("recipient.RecipientProfile", **TEST_RECIPIENT_PROFILES[recipient_id])

    expected_parent_id = "a52a7544-829b-c925-e1ba-d04d3171c09a-P"
    parent_hash = expected_parent_id[:-2]
    baker.make("recipient.RecipientLookup", **TEST_RECIPIENT_LOOKUPS[parent_hash])

    expected_name = TEST_RECIPIENT_LOOKUPS[parent_hash]["legal_business_name"]
    expected_duns = TEST_RECIPIENT_LOOKUPS[parent_hash]["duns"]
    parents = recipients.extract_parents_from_hash(recipient_hash)
    assert expected_duns == parents[0]["parent_duns"]
    assert expected_name == parents[0]["parent_name"]
    assert expected_parent_id == parents[0]["parent_id"]

@pytest.mark.django_db
def test_extract_parent_from_hash_failure():
    """Testing extracting parent duns/name from recipient hash but with recipient lookup removed
    as there may be cases where the parent recipient is not found/listed
    """
    # This one specifically has to be a child
    recipient_id = "acb93cfc-e4f8-ecd5-5ac3-fa62f115e8f5-C"
    recipient_hash = TEST_RECIPIENT_PROFILES[recipient_id]["recipient_hash"]
    baker.make("recipient.RecipientProfile", **TEST_RECIPIENT_PROFILES[recipient_id])

    expected_name = None
    expected_duns = None
    expected_parent_id = None
    parents = recipients.extract_parents_from_hash(recipient_hash)
    assert expected_duns == parents[0]["parent_duns"]
    assert expected_name == parents[0]["parent_name"]
    assert expected_parent_id == parents[0]["parent_id"]

@pytest.mark.django_db
def test_extract_location_success():
    """ Testing extracting location data from recipient hash using the DUNS table """
    recipient_hash = "a52a7544-829b-c925-e1ba-d04d3171c09a"
    baker.make("recipient.RecipientLookup", **TEST_RECIPIENT_LOOKUPS[recipient_hash])
    country_code = TEST_RECIPIENT_LOCATIONS[recipient_hash]["country_code"]
    baker.make("references.RefCountryCode", **TEST_REF_COUNTRY_CODE[country_code])

    additional_blank_fields = ["address_line3", "foreign_province", "county_name", "foreign_postal_code"]
    expected_location = TEST_RECIPIENT_LOCATIONS[recipient_hash].copy()
    expected_location["country_name"] = TEST_REF_COUNTRY_CODE[country_code]["country_name"]
    for additional_blank_field in additional_blank_fields:
        expected_location[additional_blank_field] = None
    for k in MAP_DUNS_TO_CONTRACT:
        expected_location[MAP_DUNS_TO_CONTRACT[k]] = expected_location[k]
        del expected_location[k]
    location = recipients.extract_location(recipient_hash)
    assert location == expected_location

@pytest.mark.django_db
def test_cleanup_location():
    """ Testing cleaning up the location data """

    # Test United States fix
    test_location = {"country_code": "UNITED STATES"}
    assert {"country_code": "USA", "country_name": None} == recipients.cleanup_location(test_location)

    # Test Country_Code
    baker.make("references.RefCountryCode", country_code="USA", country_name="UNITED STATES")
    test_location = {"country_code": "USA"}
    assert {"country_code": "USA", "country_name": "UNITED STATES"} == recipients.cleanup_location(test_location)

    # Test Congressional Codes
    test_location = {"congressional_code": "CA13"}
    assert {"congressional_code": "13"} == recipients.cleanup_location(test_location)
    test_location = {"congressional_code": "13.0"}
    assert {"congressional_code": "13"} == recipients.cleanup_location(test_location)

@pytest.mark.django_db
def test_extract_business_categories(monkeypatch):
    """ Testing extracting business categories from the recipient name/duns """
    recipient_hash = "a52a7544-829b-c925-e1ba-d04d3171c09a"
    recipient_name = TEST_RECIPIENT_LOOKUPS[recipient_hash]["legal_business_name"]
    recipient_uei = TEST_RECIPIENT_LOOKUPS[recipient_hash]["uei"]
    business_categories = ["le", "business", "cat"]

    utm_objects = Mock()
    utm_objects.filter().order_by().values().first.return_value = {"business_categories": business_categories}
    monkeypatch.setattr("usaspending_api.search.models.TransactionSearch.objects", utm_objects)

    baker.make("recipient.RecipientLookup", **TEST_RECIPIENT_LOOKUPS[recipient_hash])

    # Mock DUNS
    # Should add 'category_business'
    baker.make("recipient.DUNS", **TEST_DUNS[TEST_RECIPIENT_LOOKUPS[recipient_hash]["duns"]])

    expected_business_cat = business_categories + ["category_business"]
    business_cat = recipients.extract_business_categories(recipient_name, recipient_uei, recipient_hash)
    # testing for equality-only, order unnecessary
    assert sorted(business_cat) == sorted(expected_business_cat)

@pytest.mark.django_db
def test_extract_business_categories_special():
    """ Testing extracting the business categories for a special case  """
    recipient_name = "MULTIPLE RECIPIENTS"
    recipient_uei = None
    recipient_hash = ""
    business_categories = recipients.extract_business_categories(recipient_name, recipient_uei, recipient_hash)
    assert business_categories == []

@pytest.mark.django_db
def test_obtain_recipient_totals_year(monkeypatch, elasticsearch_transaction_index):
    """ Testing recipient totals with different year values """
    # Testing with specific child
    recipient_id = "acb93cfc-e4f8-ecd5-5ac3-fa62f115e8f5-C"
    recipient_hash = recipient_id[:-2]

    # load all of the transactions
    transaction_recipient_data = {
        "recipient_name": "CHILD RECIPIENT",
        "recipient_name_raw": "CHILD RECIPIENT",
        "recipient_unique_id": "000000002",
        "recipient_uei": "BBBBBBBBBBBB",
        "parent_recipient_unique_id": "000000001",
        "recipient_hash": recipient_hash,
        "recipient_levels": ["C"],
    }
    create_transaction_test_data([transaction_recipient_data] * len(TEST_SUMMARY_TRANSACTION_RECIPIENT))

    # load recipient lookup
    create_recipient_lookup_test_data(TEST_RECIPIENT_LOOKUPS[recipient_hash])

    # For latest transaction's we're pulling from recipient_profile
    associated_recipient_profile = TEST_RECIPIENT_PROFILES[recipient_id].copy()
    associated_recipient_profile["last_12_months"] = 100
    associated_recipient_profile["last_12_months_count"] = 1
    baker.make("recipient.RecipientProfile", **associated_recipient_profile)

    setup_elasticsearch_test(monkeypatch, elasticsearch_transaction_index)

    # Latest
    results = recipients.obtain_recipient_totals(recipient_id, year="latest")
    assert results[0]["total_obligation_amount"] == 100
    assert results[0]["total_obligation_count"] == 2
    assert results[0]["total_face_value_loan_amount"] == 1000
    assert results[0]["total_face_value_loan_count"] == 1

    # All
    results = recipients.obtain_recipient_totals(recipient_id, year="all")
    assert results[0]["total_obligation_amount"] == 350
    assert results[0]["total_obligation_count"] == 6
    assert results[0]["total_face_value_loan_amount"] == 3500
    assert results[0]["total_face_value_loan_count"] == 3

    # FY2016
    results = recipients.obtain_recipient_totals(recipient_id, year="2016")
    assert results[0]["total_obligation_amount"] == 50
    assert results[0]["total_obligation_count"] == 2
    assert results[0]["total_face_value_loan_amount"] == 500
    assert results[0]["total_face_value_loan_count"] == 1

@pytest.mark.django_db
def test_obtain_recipient_totals_parent(monkeypatch, elasticsearch_transaction_index):
    """ Testing recipient totals with parent child relationships """
    # Testing with specific parent/child ids
    parent_id = "a52a7544-829b-c925-e1ba-d04d3171c09a-P"

    # load recipient profiles
    for recipient_id, recipient_profile in TEST_RECIPIENT_PROFILES.items():
        baker.make("recipient.RecipientProfile", **recipient_profile)

    # load transactions for each child and parent (making sure it's excluded)
    create_transaction_test_data()

    # load recipient lookup
    create_recipient_lookup_test_data(*TEST_RECIPIENT_LOOKUPS.values())

    setup_elasticsearch_test(monkeypatch, elasticsearch_transaction_index)

    results = recipients.obtain_recipient_totals(parent_id, year="all")
    assert results[0]["total_obligation_amount"] == 150
    assert results[0]["total_obligation_count"] == 4
    assert results[0]["total_face_value_loan_amount"] == 1500
    assert results[0]["total_face_value_loan_count"] == 2

def recipient_overview_endpoint(id, year="latest"):
    endpoint = "/api/v2/recipient/duns/{}/".format(id)
    if year:
        endpoint = "{}?year={}".format(endpoint, year)
    return endpoint

@pytest.mark.django_db
def test_recipient_overview(client, monkeypatch, elasticsearch_transaction_index):
    """ Testing a simple example of the endpoint as a whole """
    r_id = "a52a7544-829b-c925-e1ba-d04d3171c09a-C"

    # Mock Transactions
    create_transaction_test_data()

    # Mock Recipient Profiles
    for recipient_id, recipient_profile in TEST_RECIPIENT_PROFILES.items():
        recipient_profile_copy = recipient_profile.copy()
        if recipient_id == r_id:
            recipient_profile_copy["last_12_months"] = 100
            recipient_profile_copy["last_12_months_count"] = 1
        baker.make("recipient.RecipientProfile", **recipient_profile_copy)

    # Mock Recipient Lookups
    create_recipient_lookup_test_data(*TEST_RECIPIENT_LOOKUPS.values())

    # Mock DUNS - should add `category-business`
    for duns, duns_dict in TEST_DUNS.items():
        test_duns_model = duns_dict.copy()
        country_code = test_duns_model["country_code"]
        baker.make("recipient.DUNS", **test_duns_model)
        baker.make("references.RefCountryCode", **TEST_REF_COUNTRY_CODE[country_code])

    setup_elasticsearch_test(monkeypatch, elasticsearch_transaction_index)

    resp = client.get(recipient_overview_endpoint(r_id))
    assert resp.status_code == status.HTTP_200_OK
    expected = {
        "name": "PARENT RECIPIENT",
        "alternate_names": [],
        "duns": "000000001",
        "uei": "AAAAAAAAAAAA",
        "recipient_id": "a52a7544-829b-c925-e1ba-d04d3171c09a-C",
        "recipient_level": "C",
        "parent_name": "PARENT RECIPIENT",
        "parent_duns": "000000001",
        "parent_id": "a52a7544-829b-c925-e1ba-d04d3171c09a-P",
        "parent_uei": "AAAAAAAAAAAA",
        "parents": [
            {
                "parent_duns": "000000001",
                "parent_id": "a52a7544-829b-c925-e1ba-d04d3171c09a-P",
                "parent_name": "PARENT RECIPIENT",
                "parent_uei": "AAAAAAAAAAAA",
            }
        ],
        "business_types": sorted(["expected", "business", "cat"] + ["category_business"]),
        "location": {
            "address_line1": "PARENT ADDRESS LINE 1",
            "address_line2": "PARENT ADDRESS LINE 2",
            "address_line3": None,
            "county_name": None,
            "city_name": "PARENT CITY",
            "congressional_code": "PARENT CONGRESSIONAL DISTRICT",
            "country_code": "PARENT COUNTRY CODE",
            "country_name": "PARENT COUNTRY NAME",
            "state_code": "PARENT STATE",
            "zip": "PARENT ZIP",
            "zip4": "PARENT ZIP4",
            "foreign_province": None,
            "foreign_postal_code": None,
        },
        "total_transaction_amount": 100,
        "total_transactions": 2,
        "total_face_value_loan_amount": 1000,
        "total_face_value_loan_transactions": 1,
    }
    # testing for equality-only, order unnecessary
    resp.data["business_types"] = sorted(resp.data["business_types"])
    assert resp.data == expected

@pytest.mark.django_db
def test_extract_hash_from_duns_or_uei_via_duns():
    """ Testing extracting the hash/name from a DUNS """
    example_duns = "000000001"
    expected_hash = "a52a7544-829b-c925-e1ba-d04d3171c09a"
    baker.make("recipient.RecipientLookup", **TEST_RECIPIENT_LOOKUPS[expected_hash])

    recipient_hash = recipients.extract_hash_from_duns_or_uei(example_duns)
    assert UUID(expected_hash) == recipient_hash

@pytest.mark.django_db
def test_extract_hash_from_duns_or_uei_via_uei():
    """ Testing extracting the hash/name from a DUNS """
    example_uei = "AAAAAAAAAAAA"
    expected_hash = "a52a7544-829b-c925-e1ba-d04d3171c09a"
    baker.make("recipient.RecipientLookup", **TEST_RECIPIENT_LOOKUPS[expected_hash])

    recipient_hash = recipients.extract_hash_from_duns_or_uei(example_uei)
    assert UUID(expected_hash) == recipient_hash

def recipient_children_endpoint(duns, year="latest"):
    endpoint = "/api/v2/recipient/children/{}/".format(duns)
    if year:
        endpoint = "{}?year={}".format(endpoint, year)
    return endpoint

@pytest.mark.django_db
def test_child_recipient_success(client, monkeypatch, elasticsearch_transaction_index):
    """ Testing successful child recipient calls """
    child1_id = "a52a7544-829b-c925-e1ba-d04d3171c09a-C"
    parent_child1_duns = "000000001"
    child2_id = "acb93cfc-e4f8-ecd5-5ac3-fa62f115e8f5-C"
    child2_duns = "000000002"

    # Mock Recipient Profiles
    create_recipient_profile_test_data(*TEST_RECIPIENT_PROFILES.values())

    # Mock Recipient Lookups
    create_recipient_lookup_test_data(*TEST_RECIPIENT_LOOKUPS.values())

    # Mock Transactions
    create_transaction_test_data()

    setup_elasticsearch_test(monkeypatch, elasticsearch_transaction_index)

    # Ignoring nonexistent child hash - 00eadaa1-6e1f-22f2-419b-e5894374db96
    child1_object = {
        "recipient_id": child1_id,
        "name": "PARENT RECIPIENT",
        "duns": parent_child1_duns,
        "uei": "AAAAAAAAAAAA",
        "amount": 100,
        "state_province": "PARENT STATE",
    }
    child2_object = {
        "recipient_id": child2_id,
        "name": "CHILD RECIPIENT",
        "duns": child2_duns,
        "uei": "BBBBBBBBBBBB",
        "amount": 50,
        "state_province": "CHILD STATE",
    }
    expected = [child1_object, child2_object]
    resp = client.get(recipient_children_endpoint(parent_child1_duns, "all"))
    assert resp.status_code == status.HTTP_200_OK
    # testing for equality-only, order unnecessary
    assert sorted(resp.data, key=lambda key: key["recipient_id"]) == expected

@pytest.mark.django_db
def test_child_recipient_failures(client):
    """ Testing failed child recipient calls """
    # Mock Recipient Profiles
    create_recipient_profile_test_data(*TEST_RECIPIENT_PROFILES.values())

    # Mock Recipient Lookups
    create_recipient_lookup_test_data(*TEST_RECIPIENT_LOOKUPS.values())

    # Mock Transactions
    create_transaction_test_data()

    # Testing for non-existent DUNS
    non_existent_duns = "000000003"
    resp = client.get(recipient_children_endpoint(non_existent_duns, "all"))
    assert resp.status_code == status.HTTP_400_BAD_REQUEST
    assert resp.data["detail"] == "Recipient not found: '{}'.".format(non_existent_duns)


import os
from dataclasses import dataclass

from flask import Flask, render_template, send_from_directory
from sqlalchemy import func, select

from src.jinja_helper import today
from src.database import get_db, init_app
from src.views import auth, books, members
from src.models import Book, Issued, Member

app = Flask(__name__)

app.config.from_pyfile("config.py", silent=True)
init_app(app)

@app.get("/")
@auth.login_required
def index():
    db = get_db()

    @dataclass
    class Status:
        book_with_copies: int = 0
        book_without_copies: int = 0
        issued_books: int = 0
        total_members: int = 0
        members_with_dept: int = 0
        total_dept: int = 0

    def db_execute(stmt):
        return db.execute(stmt).scalar()

    status = Status()
    status.book_with_copies = db_execute(select(func.sum(Book.total)))
    status.book_without_copies = db_execute(select(func.count(Book.bookID)))
    status.issued_books = db_execute(
        select(func.count(Issued.id)).where(Issued.return_date == None)
    )

    status.total_members = db_execute(select(func.count(Member.id)))
    status.members_with_dept = db_execute(
        select(func.count(Member.id)).where(Member.dept > 0)
    )
    status.total_dept = db_execute(select(func.sum(Member.dept)))

    return render_template("index.html", status=status)

@app.route("/favicon.ico")
def favicon():
    return send_from_directory(os.path.join(app.root_path, "static"), "favicon.ico")

app.jinja_env.globals["today"] = today

app.register_blueprint(auth.bp)
app.register_blueprint(books.bp)
app.register_blueprint(members.bp)


import numpy as np
import cv2 as cv

class FrameWrapper:
    def __init__(self,line_down,line_middle,line_up,frameWidth) -> None:
        #using hight of lines position and width of frame to create point to plot lines on a screen
        self.line_down = np.array([[0, line_down],[frameWidth, line_down]], np.int32)
        self.line_down = self.line_down.reshape((-1,1,2))
        self.line_up = np.array([[0, line_up],[frameWidth, line_up]], np.int32)
        self.line_up = self.line_up.reshape((-1,1,2))
        self.line_middle= np.array([[0, line_middle],[frameWidth, line_middle]], np.int32)
        self.line_middle = self.line_middle.reshape((-1,1,2))

    def __call__(self, frame, output_counted,already_counted):
        #plot lines to make solution easier to understand
        cv.polylines(frame,[self.line_down],False,[255,0,0],thickness=2)
        cv.polylines(frame,[self.line_up],False,[0,0,255],thickness=2)
        cv.polylines(frame,[self.line_middle],False,[255,255,255],thickness=2)
        #add info on a frame
        str_on_frame = "Total Counting = %d, Already Counted = %d" % (output_counted,already_counted)
        cv.putText(frame, str_on_frame, (5,60), cv.FONT_HERSHEY_SIMPLEX ,2, (0,0,255), 2, cv.LINE_AA)

        return frame


from unittest.mock import patch, MagicMock

import pytest
from aiohttp.test_utils import AioHTTPTestCase, unittest_run_loop
from aiohttp import web

from main import get_app, post_to_slack

hub_payload = """{
  "callback_url": "https://registry.hub.docker.com/u/svendowideit/testhook/hook/2141b5bi5i5b02bec211i4eeih0242eg11000a/",
  "push_data": {
    "images": [
        "27d47432a69bca5f2700e4dff7de0388ed65f9d3fb1ec645e2bc24c223dc1cc3",
        "51a9c7c1f8bb2fa19bcd09789a34e63f35abb80044bc10196e304f6634cc582c",
        "..."
    ],
    "pushed_at": 1417566161,
    "pusher": "trustedbuilder",
    "tag": "latest"
  },
  "repository": {
    "comment_count": "0",
    "date_created": 1417494799,
    "description": "",
    "dockerfile": "Dockerfile detail",
    "full_description": "Docker Hub based automated build from a GitHub repo",
    "is_official": false,
    "is_private": true,
    "is_trusted": true,
    "name": "testhook",
    "namespace": "svendowideit",
    "owner": "svendowideit",
    "repo_name": "svendowideit/testhook",
    "repo_url": "https://registry.hub.docker.com/u/svendowideit/testhook/",
    "star_count": 0,
    "status": "Active"
  }
}
"""

class MyAppTestCase(AioHTTPTestCase):

    async def get_application(self):
        return get_app()

    @unittest_run_loop
    async def test_index(self):
        resp = await self.client.request("GET", "/")
        assert resp.status == 200
        text = await resp.text()
        assert "I" in text

    @unittest_run_loop
    async def test_dockerhub(self):

        with patch("main.post_to_slack") as mock:
            resp = await self.client.request("POST", "/dockerhub?slack_webhook=https://hooks.slack.com/services/spam/ham/egggggggggggggs", data=hub_payload)
        mock.assert_called_once_with("latest", "trustedbuilder", "svendowideit/testhook", "https://registry.hub.docker.com/u/svendowideit/testhook/", "https://hooks.slack.com/services/spam/ham/egggggggggggggs")
        assert resp.status == 200
        text = await resp.text()
        assert "OK" in text

    def test_slack_message_should_parse_to_json(self):
        with patch("main.requests") as mock:
            post_to_slack("tag", "pusher", "reponame", "repourl", "hookurl")
            assert "hookurl" == mock.post.call_args[0][0]
            data = mock.post.call_args[1]["data"]["payload"]
            import json
            try:
                json.loads(data)
            except json.decoder.JSONDecodeError:
                pytest.fail("Payload cannot be parsed")


#!/usr/bin/env python
from pprint import pprint
import copy
import os
import re
import numpy as np
from matplotlib import pyplot as plt

from coffea import hist

from bucoffea.plot.util import (acc_from_dir, merge_datasets, merge_extensions,
                                scale_xs_lumi)
from bucoffea.helpers.paths import bucoffea_path
import uproot
pjoin = os.path.join

data_err_opts = {
        'linestyle':'none',
        'marker': '.',
        'markersize': 10.,
        'color':'k',
        'elinewidth': 1,
        'emarker': '_'
    }

def get_old_kfac(tag):
    if tag.startswith('w'):
        f = uproot.open(bucoffea_path('data/sf/theory/merged_kfactors_wjets.root'))
    elif tag.startswith('dy'):
        f = uproot.open(bucoffea_path('data/sf/theory/merged_kfactors_wjets.root'))
    return f['kfactor_monojet_qcd']
def plot_lhe_v_pt(acc, tag, regex, outputrootfile, pttype):
    outdir = './output/'
    if not os.path.exists(outdir):
        os.makedirs(outdir)

    fig, (ax, rax) = plt.subplots(2, 1, figsize=(7,7), gridspec_kw={"height_ratios": (3, 1)}, sharex=True)
    # new_ax = hist.Bin('vpt','LHE V $p_{T}$ (GeV)',list(range(100,500,50)) + list(range(500,1000,100)) + list(range(1000,2000,250)))
    new_ax = hist.Bin('vpt','LHE V $p_{T}$ (GeV)',list(range(80,800,40))+list(range(800,2000,100)))

    for dist in ['gen_vpt']:
        h = copy.deepcopy(acc[dist])
        h = h.integrate('type',pttype)
        h = h.rebin(h.axis('vpt'), new_ax)
        h = merge_extensions(h, acc, reweight_pu=False)
        scale_xs_lumi(h)
        h = merge_datasets(h)
        h = h[re.compile(regex)]
        h = h.integrate('weight_type','nominal')
        h = h.integrate('weight_index',slice(-0.5,0.5))
        hist.plot1d(
            h,
            overlay='dataset',
            overflow='all',
            binwnorm=True,
            ax=ax)
        lo = h[re.compile('.*HT.*')].integrate('dataset')
        nlo = h[re.compile('.*LHE.*')].integrate('dataset')

        hist.plotratio(nlo, lo,
            ax=rax,
            denom_fill_opts={},
            guide_opts={},
            unc='num',
            overflow='all',
            error_opts=data_err_opts,
            label='2017 NLO/LO ratio'
            )
        old = get_old_kfac(tag)
        old_x = 0.5*(old.bins[:,0]+old.bins[:,1])
        rax.plot(old_x, old.values,'ob-', label='2016 QCD k fac')
        rax.plot(old_x, old.values * pdfwgt_sf(old_x),'or-', label='2016 x ad-hoc DY pdfwgt SF')
        ax.set_yscale('log')
        ax.set_ylim(1e-3,1e6)
        rax.set_ylim(0,2)
        rax.legend()

        fig.savefig(pjoin(outdir,f'{tag}_{dist}.pdf'))

        sf_x = lo.axis('vpt').edges()
        sf_y = nlo.values()[()] / lo.values()[()]

        # try:
        #     f = uproot.create(f'gen_v_pt_qcd_sf.root')
        # except OSError:

        outputrootfile[tag] = (sf_y,sf_x)

def pdfwgt_sf(vpt):
    return 1/(1.157 + 2.291e-4 * vpt + 6.0612e-7 * vpt**2)

# def sf_comparison(acc, tag, regex):
#     outdir = './output/'
#     if not os.path.exists(outdir):
#         os.makedirs(outdir)

#     fig = plt.gcf()
#     # new_ax = hist.Bin('vpt','LHE V $p_{T}$ (GeV)',list(range(100,500,50)) + list(range(500,1000,100)) + list(range(1000,2000,250)))
#     new_ax = hist.Bin('vpt','LHE V $p_{T}$ (GeV)',list(range(80,2000,40)))

#     for dist in ['gen_vpt']:
#         fig.clf()
#         h = copy.deepcopy(acc[dist])
#         h = h.rebin(h.axis('vpt'), new_ax)
#         h = merge_extensions(h, acc, reweight_pu=False)
#         scale_xs_lumi(h)
#         h = merge_datasets(h)
#         h = h[re.compile(regex)]
#         h = h.integrate('weight_type','nominal')
#         h = h.integrate('weight_index',slice(-0.5,0.5))

#         lo = h[re.compile('.*HT.*')].integrate('dataset')
#         nlo = h[re.compile('.*LHE.*')].integrate('dataset')

#         x_sf = lo.axis('vpt').identifiers()
#         y_sf = nlo.values () / lo.values()

#         old = get_old_kfac(tag)

#         rax.plot(0.5*(old.bins[:,0]+old.bins[:,1]), old.values,'or-', label='2016 QCD k fac')
#         ax.set_yscale('log')
#         ax.set_ylim(1e-3,1e6)
#         rax.set_ylim(0,2)
#         rax.legend()

#         fig.savefig(pjoin(outdir,f'{tag}_{dist}.pdf'))
def main():
    acc = acc_from_dir("./input/das_lhevpt_v3")
    outputrootfile = uproot.recreate(f'2017_gen_v_pt_stat1_qcd_sf.root')
    plot_lhe_v_pt(acc, tag='wjet_dilep', regex='W.*',pttype='dilepton',outputrootfile=outputrootfile)
    plot_lhe_v_pt(acc, tag='dy_dilep', regex='.*DY.*',pttype='dilepton',outputrootfile=outputrootfile)

    outputrootfile = uproot.recreate(f'2017_gen_v_pt_lhe_qcd_sf.root')
    plot_lhe_v_pt(acc, tag='wjet_nano', regex='W.*',pttype='nano',outputrootfile=outputrootfile)
    plot_lhe_v_pt(acc, tag='dy_nano', regex='.*DY.*',pttype='nano',outputrootfile=outputrootfile)

if __name__ == "__main__":
    main()


import sqlite3

try:
    con=sqlite3.connect("C:\\Users\\Balarubinan\\PycharmProjects\\SocialMediaApp\\src\\storeDB.db")
    cur=con.cursor()
    # con.row_factory=sqlite3.Row
except:
    print("DB connect error!")

def add_comment(post_id,comment,username):
    try:
        cur.execute(f"insert into Comment values('{post_id}','{comment}','{username}')")
        con.commit()
    except:
        print("Comment add error!")

def delete_comment(post_id,username):
    try:
        cur.execute(f"delete from Comment where post_id='{post_id}' and username='{username}'")
        con.commit()
    except:
        print("Delete error!")

def add_to_friends(frnd1,frnd2):
    try:
        cur.execute(f"insert into Friends values('{frnd1}','{frnd2}')")
        con.commit()
    except:
        print("add friends error!")

def del_from_friends(frnd1,frnd2):
    try:
        cur.execute(f"delete from Friends where frnd1='{frnd1}' and frnd2='{frnd2}")
        cur.execute(f"delete from Friends where frnd1='{frnd2}' and frnd2='{frnd1}")
        con.commit()
    except:
        print("Delete friends error!")

def fetch_friends(given_name):
    lis=[]
    try:
        cur.execute(f"select frnd1 from Friends where frnd2='{given_name}'")
        lis.append(cur.fetchall())
        cur.execute(f"select frnd2 from Friends where frnd1='{given_name}'")
        lis.append(cur.fetchall())
        return lis
    except:
        print("search friends error!")

def add_image(id,posted_by,type,path):
    try:
        cur.execute(f"insert into Images values(?,?,?,?)",(id,posted_by,type,path))
        con.commit()
    except:
        print("add_image error!")

def del_image(id):
    try:
        cur.execute(f"delete from Images where id='{id}'")
        con.commit()
    except:
        print("Error del_image")

# explain after here

def add_post(post_id,posted_by,type,caption,likes,images_id):
    try:
        cur.execute("insert into Post values(?,?,?,?,?,?)",(post_id,posted_by,type,caption,likes,images_id))
        con.commit()
    except:
        print("Add post error!")

def delete_post(post_id):
    try:
        cur.execute(f"delete from Post where post_id='{post_id}'")
        con.commit()
    except:
        print("Post delete error")

def fetch_user_posts(frnd_list):
    try:
        lis=[]
        for x in frnd_list:
            cur.execute(f"select * from Post where posted_by='{x}' or type='public'")
            lis.extend(cur.fetchall())
    except:
        print("Post fetch error!")

def add_user(email,username,password,profile_pic_link):
    try:
        cur.execute("insert into User values(?,?,?,?)",(email,username,password,profile_pic_link))
        con.commit()
    except:
        print("User add error")

def del_user(email):
    try:
        cur.execute(f"delete from User where email='{email}'")
    except:
        print("User del error")

def fetch_user_details(email):
    try:
        cur.execute(f"select * from User where email='{email}'")
        return cur.fetchall()
    except:
        print("User fetch error!")

def fetch_password(email):
    try:
        cur.execute(f"select password from User where email='{email}'")
        return cur.fetchall()[0] # 0 contains the password!
    except:
        print("User fetch pass error!")

# new functions !

def increase_like(post_id):
    try:
        cur.execute(f"update Post set likes=likes+1 where post_id='{post_id}'")
        con.commit()
    except:
        print("Update error!")

def fetch_post_by_id(post_id):
    try:
        cur.execute(f"select * from Post where post_id='{post_id}'")
        return cur.fetchall()[0]
    except:
        print("Update error!")

def fetch_comments(post_id):
    try:
        cur.execute(f"select * from comment where post_id='{post_id}'")
        return cur.fetchall()
    except:
        print("fetch comment error error!")

# type in the stuff for the request DB okay??
# don't forget to explain about auto commit

# add_user('b@gmail','Bala','jjj','acadc')


import shutil
import tempfile

from django.contrib.auth import get_user_model
from django.core.files.uploadedfile import SimpleUploadedFile
from django.test import Client, TestCase, override_settings
from django.urls import reverse

from ..models import Group, Post

MEDIA_ROOT = tempfile.mkdtemp()
User = get_user_model()

def gif_create():
    small_gif = (
        b'\x47\x49\x46\x38\x39\x61\x02\x00'
        b'\x01\x00\x80\x00\x00\x00\x00\x00'
        b'\xFF\xFF\xFF\x21\xF9\x04\x00\x00'
        b'\x00\x00\x00\x2C\x00\x00\x00\x00'
        b'\x02\x00\x01\x00\x00\x02\x02\x0C'
        b'\x0A\x00\x3B'
    )
    uploaded = SimpleUploadedFile(
        name='test_small.gif',
        content=small_gif,
        content_type='image/gif'
    )
    return uploaded

@override_settings(MEDIA_ROOT=MEDIA_ROOT)
class TaskCreateFormTests(TestCase):
    @classmethod
    def tearDownClass(cls):
        shutil.rmtree(MEDIA_ROOT, ignore_errors=True)
        super().tearDownClass()

    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        cls.author = User.objects.create_user(username='author')

        cls.group = Group.objects.create(
            title='test_title',
            description='test_description',
            slug='test_slug'
        )

        cls.post = Post.objects.create(
            text='test_post',
            author=cls.author,
            group=cls.group,
            image=gif_create()
        )

    def setUp(self):
        self.guest_client = Client()
        self.authorized_client = Client()
        self.authorized_client.force_login(self.author)

    def test_create_post(self):
        post_count = Post.objects.count()
        form_data = {
            'text': 'test_text',
            'image': gif_create(),

        }
        response = self.authorized_client.post(
            reverse('posts:post_create'),
            data=form_data,
            follow=True
        )
        self.assertRedirects(response, reverse(
            'posts:profile', kwargs={'username': self.author}))
        self.assertEqual(Post.objects.count(), post_count + 1)

        self.assertTrue(
            Post.objects.filter(
                text='test_post',
                image='posts/test_small.gif'
            ).exists()
        )

    def test_edit_post(self):
        post_count = Post.objects.count()

        form_data = {
            'text': 'test_text',
            'group': self.__class__.group.pk,
            'image': gif_create(),

        }
        response = self.authorized_client.post(
            reverse('posts:post_edit', kwargs={'pk': self.post.pk}),
            data=form_data,
            follow=True
        )
        self.assertRedirects(response, reverse(
            'posts:post_detail', kwargs={'post_id': self.post.id}))
        self.assertEqual(Post.objects.count(), post_count)


# -*- coding: utf-8 -*-
#!/usr/bin/env python3
'''
Created on Sat Sep 15 19:31:52 CST 2018
@Mail: minnglee@163.com
@Author: Ming Li
'''

import sys,os,logging,click,re

logging.basicConfig(filename='{0}.log'.format(os.path.basename(__file__).replace('.py','')),
                    format='%(asctime)s: %(name)s: %(levelname)s: %(message)s',level=logging.DEBUG,filemode='w')
logging.info(f"The command line is:\n\tpython3 {' '.join(sys.argv)}")

def LoadSeq(File):
    seq = ''
    for line in File:
        line = line.strip()
        if line[0] == '>':
            Chr = line.split()[0][1:]
            continue
        else: seq += line
    return Chr,seq
@click.command()
@click.option('-i','--input',type=click.File('r'),help='The input file',required=True)
#@click.option('-o','--output',type=str,help='The prefix of output file',required=True)
def main(input):
    Chr,Seq = LoadSeq(input)
    Mask = [(m.start(0), m.end(0)) for m in re.finditer('1*', Seq)]
    Alignable = [(m.start(0), m.end(0)) for m in re.finditer('0*', Seq)]
    MaskFile = open(f'{Chr}.mask.region','w')
    for region in Mask:
        if int(region[1]) - int(region[0]) == 0: continue
        MaskFile.write(f'{Chr}\t{int(region[0])+1}\t{int(region[1])}\n')
    AlignableFile = open(f'{Chr}.alignable.region','w')
    for region in Alignable:
        if int(region[1]) - int(region[0]) == 0: continue
        AlignableFile.write(f'{Chr}\t{int(region[0])+1}\t{int(region[1])}\n')
if __name__ == '__main__':
    main()


import pandas
import numpy as np
from sklearn.cross_validation import KFold
from sklearn.ensemble import RandomForestRegressor
from sklearn.grid_search import GridSearchCV

def main():
    data = pandas.read_csv('abalone.csv')
    data['Sex'] = data['Sex'].map(lambda x:
                                  1 if x == 'M' else (-1 if x == 'F' else 0))
    features = np.asarray(data.ix[:, :'ShellWeight'])
    markers = np.asarray(data['Rings'])
    grid = {'n_estimators': range(1, 51)}
    cv = KFold(n=len(markers), n_folds=5, shuffle=True, random_state=1)
    reg = RandomForestRegressor(random_state=1)
    gs = GridSearchCV(reg, grid, scoring='r2', cv=cv)
    gs.fit(features, markers)

    means = [score_tuple[1] for score_tuple in gs.grid_scores_]
    n_estimators = np.array(np.where(np.asarray(means) > 0.52)) + 1
    print(n_estimators[0][0])

if __name__ == "__main__":
    main()


'''
This is the code for the Yo-Yo
Simulated Model written by
Rohan Deswal, 20202UMP1071, MPAE, NSUT Delhi
based on the work of Koichi Hashimoto and Toshiro Noritsugu
of Okayama University.
'''

import numpy as np
import scipy as sp
import matplotlib.pyplot as plt

m = 1 # mass of the Yo-Yo
r = 0.1 # radius of the Yo-Yo
l = 1 # lenght of the complete string
e_friction = 0.15 # approx coefficent of friction between common plastics and cotton strings
I = 0.5*m*(r**2) # inertia fo the Yo-Yo
g = 9.801 # gravitational acceleration

dt = np.pi/100 # Time Interval
time_axis = np.arange(0,10,0.1)
theta = np.arange(0,np.pi,dt) # Input of Various Angular Positions
theta_d1 = np.gradient(np.sin(theta),dt) # Velocity profile for a sinusoidal input rotation
theta_d2 = np.gradient(theta_d1,dt) # corresponding angular acceleration

def linear_acceleration(theta_d1_val, theta_d2_val):
    '''
    This function is used to calculate the instantaneous
    linear acceleration of the Yo-Yo given the angular
    acceleration and angular velocity of the Yo-Yo based on
    the analysis given in the document.
    '''
    return (((I+m*(r**2))*theta_d2_val + r*e_friction*theta_d1_val)/(m*r)) - g

h_d2 = [-linear_acceleration(theta_d1[i],theta_d2[i]) for i in range(len(theta_d1))]
h_d1 = [0]
h = [0]

# Calculation of Integral to find Height of the Yo-Yo
for a in h_d2:
    h_d1.append(h_d2[-1] + a*dt)
    h.append(100*(h_d1[-1] + 0.5*a*dt**2) % 1000)

fig, axs = plt.subplots(2, 2)
axs[0, 0].plot(time_axis, theta)
axs[0, 0].set_title('Input Angles or Value of Rotation')
axs[0, 1].plot(time_axis, theta_d1, 'tab:orange')
axs[0, 1].set_title('Angular Velocity or the first derivative of θ')
axs[1, 0].plot(time_axis, theta_d2, 'tab:green')
axs[1, 0].set_title('Angular Acceleration or the second derivative of θ')
axs[1, 1].plot(time_axis, h[1:], 'tab:red')
axs[1, 1].set_title('Value height of the Yo-Yo in cm')

fig.tight_layout()

for ax in axs.flat:
    ax.set(xlabel='time in seconds')

plt.legend(loc="best")
plt.show()


#<---------Solucion a los problemas propuestos en el notebook--------------->
#<------------------01 escalares vectores matrices tensores.ipynb-------------->
import numpy as np
#<--------------Crear un escalar 42-------------->
a=42
print("Esto es un escalar con valor 42\n",a)
#<------------Crear el vector con los elementos 2,3,5,7---------------->
a=np.array([2,3,5,7])
print("Esto es un vector con elementos 2,3,5,7\n",a)
#<Crear una matriz de 2x2 con los elementos 2,3 en la primera fila y 5,7 en la segunda fila>
a=np.array([[2,3],[5,7]])
print("Esto una matriz de 2x2 con los elementos 2,3 en la primera fila y 5,7 en la segunda fila\n",a)
#<Crear un tensor que al graficarlo veamos Blanco, Negro, Gris, desde arriba hacia abajo.>
#<-----------Nota------------>
#<-----La funcion imshow de matplot realiza el "plot" de pixeles, este normalmente tiene valores en rgb--------->
#<-------Blaco en rgb (255,255,255)---------->
#<--------Gris en rgb (128,128,128)----------->
#<----------Negro en rgb (0,0,0)--------------->
#<----------- el tensor que formaremos tendra dimensiones de 3x3------------->
a=np.array([
    [[255,255,255],[255,255,255],[255,255,255]], #Blanco
    [[128,128,128],[128,128,128],[128,128,128]], # Gris
    [[0,0,0],[0,0,0],[0,0,0]] # Negro
]
)
#<--------------Libreria para plotear------------>
import matplotlib.pyplot as plt
#<-------------Deshabilito los ejes---------->
plt.yticks([]);plt.xticks([])
plt.imshow(a)
plt.show()


import pandas as pd
import dash
import dash_html_components as html
import dash_core_components as dcc
from dash.dependencies import Input, Output, State
import plotly.graph_objects as go
import plotly.express as px
from dash import no_update

app = dash.Dash(__name__)

# REVIEW1: Clear the layout and do not display exception till callback gets executed
app.config.suppress_callback_exceptions = True

# Read the automobiles data into pandas dataframe
auto_data =  pd.read_csv('automobileEDA.csv', encoding = "ISO-8859-1")
print(auto_data.head(2))

#Layout Section of Dash
app.layout = html.Div(
    children=[
        html.H1('Car Automobile Components', style={'textAlign':'center', 'color':'#503D36', 'font-size':24}),
        # Next Div Level
        html.Div([
            # Inner Division for adding dropdown helper text
            html.Div(
                html.H2('Drive Wheels Type : ', style={'margin-right':'2em'}) # don't see a need for a comma here
            ),
            # Inner Division
            dcc.Dropdown(
                id='demo-dropdown',
                options=[
                    {'label':'Rear Wheel Drive', 'value':'rwd'},
                    {'label':'Front Wheel Drive', 'value':'fwd'},
                    {'label':'Four Wheel Drive', 'value':'4wd'}
                ],
                value='rwd'
            ),
            html.Div([
                html.Div([], id='plot1'),
                html.Div([], id='plot2')
            ], style={'display':'flex'}) # another comma at the eol here in template and unsure if needed
        ])
    ]
)

# Place to add @app.callback Decorator
# The inputs and outputs of our application's interface are described declaratively as the arguments of @app.callback decorator.
# Dash - inputs/outputs of app are properties of a particular components
# In this example, our input is the value property of the component that has the ID demo-dropdown
# Our layout has 2 outputs so we need to create 2 output components.
# Here, the component property will be children as we have created empty division and passing in dcc.Graph (figure) after computation.
# TASK 3E
@app.callback(
    [
        Output(component_id='plot1', component_property='children'),
        Output(component_id='plot2', component_property='children')
    ],
    Input(component_id='demo-dropdown', component_property='value')
)
# Place to define the callback function .
# Whenever an input property changes, the function that the callback decorator wraps will get called automatically.
# TASK 3F
def display_selected_drive_charts(value):
    # Subset dataframe to selected drive wheels value from dropdown
    filtered_df = auto_data[auto_data['drive-wheels'] == value].groupby(['drive-wheels', 'body-style'], as_index=False)['price'].mean()
    filtered_df = filtered_df
    fig1 = px.pie(filtered_df, values='price', names='body-style', title='Pie Chart')
    fig2 = px.bar(filtered_df, x='body-style', y='price', title='Bar Chart')
    return [
        dcc.Graph(figure=fig1),
        dcc.Graph(figure=fig2)
    ]

if __name__ == '__main__':
    app.run_server()


"""Snooper is a data processor created for the purpose of downloading JSON data from supported
    WeedMaps APIs to be converted into Pandas DataFrames and stored in multiple different formats.
    1. Objects
        1. data_dir
            os.path data directory used for saving / exporting data from snooper.
        2. save_file
            the primary save file for snooper to export to.
        3. export_file
            an optional test export file for snooper to export to.
    2. Classes
        1. Snooper
            the primary application class for snooper.
"""
import json
import os.path
from datetime import datetime
from pathlib import Path
from lib import actors
from lib import util

# Data directory
data_dir = os.path.dirname(os.path.abspath(__file__))+"/data"
Path(data_dir).mkdir(parents=True, exist_ok=True)

# Primary save file
save_file = data_dir+"/snooper.json"

# debug save file
export_file = data_dir+"/export.json"

class Snooper:
    """
    A class used to represent the primary application of the snooper package.

    Attributes
    ----------
    data_lib : dict
        dictionary representing data stored from converted json received from WeedMaps.

    app : Object/Snooper
        object representing the class executable of Snooper. only accessed if this file is
        ran as main.

    Regions : Object/actors.WMRegions
        Data processor for WeedMaps Regions

    SubRegions : Object/actors.WMSubRegions
        Data processor for WeedMaps SubRegions

    Dispensaries : Object/actors.WMDispensaries
        Data processor for WeedMaps dispensary listings

    Menus : Object/actors.WMMenus
        Data processor for WeedMaps listing menu items

    Deals : Object/actors.WMDeals
        Data processor for WeedMaps deals

    Pandas : Object/util.SnooperToPandas
        addon object that provides additional support via methods for pandas

    selected_region : dict
        dictionary of data extracted using matching method.

    selected_subregion : dict
        dictionary of data extracted using matching method.

    selected_listing : dict
        dictionary of data extracted using matching method.

    selected_menu : dict
        dictionary of data extracted using matching method.

    selected_item : dict
        dictionary of data extracted using matching method.

    Methods
    -------
    select_region(region_slug)
        Stores region data from data_lib in memory using region_slug as key.

    select_subregion(subregion_slug)
        Stores subregion data from data_lib in memory using subregion_slug as key.

    select_listing(listing)
        Stores listing data from data_lib in memory using listing_slug as key.

    select_menu()
        Stores menu data from data_lib in memory using the stored listing data from select_listing.

    load_json(file)
        Loads json from file into data_lib

    save_json(file)
        Saves data from data_lib as json into file

    main()
        Primary standalone executable function of snooper
    """
    data_lib = {}

    def __init__(self):
        """
        Constructs all the necessary attributes for the Snooper object.

        Parameters
        ----------
        None

        """
        self.Regions=actors.WMRegions(self)
        self.SubRegions = actors.WMSubRegions(self)
        self.Dispensaries = actors.WMDispensaries(self)
        self.Menus = actors.WMMenus(self)
        self.Deals = actors.WMDeals(self)
        self.Pandas = util.SnooperToPandas(self)
        self.selected_region = None
        self.selected_subregion = None
        self.selected_listing = None
        self.selected_menu = None
        self.selected_item = None
        print("Creating Snooper App") # DEBUG

    def select_region(self, region_slug):
        """
        Selects a region using region_slug as a key for data_lib

        Parameters
        ----------
        region_slug : str
            A string representing the key of the region to extract from data_lib.

        Returns
        -------
        None
        """
        try:
            self.selected_region = self.data_lib[region_slug]
            print(f"Selected region: {region_slug}")
        except KeyError:
            print("Sorry, but that region doesn't appear to be loaded. \
                Refresh region data and try again.")

    def select_subregion(self, subregion_slug):
        """
        Selects a subregion using subregion_slug as a key for selected_region

        Parameters
        ----------
        subregion_slug : str
            A string representing the key of the subregion to extract from selected_region.

        Returns
        -------
        None
        """
        try:
            self.selected_subregion = self.selected_region[subregion_slug]
            print(f"Selected subregion: {subregion_slug}")
        except KeyError:
            print("Sorry, but that subregion doesn't appear to be loaded. \
                Refresh subregion and try again.")
        except TypeError:
            print("Sorry, but you must select a region first before attempting to select \
                a subregion.")

    def select_listing(self, listing_slug):
        """
        Selects a dispensary listing using listing_slug as a key for selected_subregion

        Parameters
        ----------
        listing_slug : str
            A string representing the key of the listing to extract from selected_subregion.

        Returns
        -------
        None
        """
        try:
            self.selected_listing = self.selected_subregion['listings'][listing_slug]
            print(f"Selected listing: {listing_slug}")
        except KeyError:
            print("Sorry, but that listing doesn't appear to be loaded. \
                Refresh subregion listings and try again.")
        except TypeError:
            print("Sorry, but you must select a subregion first before attempting to \
                select a listing.")

    def select_menu(self):
        """
        Selects the menu listing from selected_listing

        Parameters
        ----------
        None

        Returns
        -------
        None
        """
        try:
            self.selected_menu = self.selected_listing['menu']
            print(f"Selected Menu: {self.selected_listing['slug']}")
        except KeyError:
            print("Sorry, but that listing doesn't appear to have a menu loaded. \
                Refresh menus for listing or subregion and try again.")
        except TypeError:
            print("Sorry, but you must select a listing first before attempting to select a menu.")

    # def select_item(self, item_slug):
    #     pass

    def load_json(self, file):
        """
        Loads json from file and saves into data_lib as dict.

        Parameters
        ----------
        file : str
            path to file that should be used for data export

        loaded : boolean
            boolean representing the load success state of the file

        Returns
        -------
        loaded : boolean
        """
        loaded = False
        try:
            # in_file = open(common.save_file, "r")
            with open(file, encoding="utf8") as in_file:
                print("Loading data from file...")
                self.data_lib = json.load(in_file)
                print("Data loaded!")
                loaded = True
        except FileNotFoundError:
            print("File not found: Starting from scratch!")
            self.data_lib={}
        return loaded

    def save_json(self, file, data):
        """
        Saves data_lib to a file as json.

        Parameters
        ----------
        file : str
            path to file that should be used for data export
        data
            dictionary to write to file as json

        Returns
        -------
        None
        """
        print("Saving JSON file")
        with open(file, "w", encoding="utf8") as out_file:
            out_file.truncate()
            json.dump(data, out_file)

    def main(self):
        """
        Primary executable function of Snooper

        Parameters
        ----------
        None

        Returns
        -------
        None
        """
        start_time = datetime.now()
        print("Running main()")
        # if self.load_json(save_file):
        #     # Define pandas DataFrames
        #     listings_frame = self.Pandas.listings()
        #     subregion_menus_frame = self.Pandas.subregion_menus()
        #     # subregion_deals_frame = self.Pandas.subregion_deals()
        #     region_deals_frame = self.Pandas.region_deals()

        #     # Export to CSV
        #     listings_frame.to_csv(data_dir + "/listings.csv")
        #     subregion_menus_frame.to_csv(data_dir + "/subregion_menus.csv")
        #     region_deals_frame.to_csv(data_dir + "/region_deals.csv")

        # else:
        # Download subregions for region
        self.Regions.get_subregions("oklahoma")

        # Select desired region and subregion
        self.select_region("oklahoma")
        self.select_subregion('oklahoma-city')

        # Download data for export
        self.SubRegions.get_listings(self.selected_subregion)
        self.Regions.get_menus()
        self.Regions.get_deals()

        # Define pandas DataFrames
        listings_frame = self.Pandas.listings()
        subregion_menus_frame = self.Pandas.subregion_menus()
        # subregion_deals_frame = self.Pandas.subregion_deals()
        region_deals_frame = self.Pandas.region_deals()

        # Export to CSV
        listings_frame.to_csv(data_dir + "/listings.csv")
        subregion_menus_frame.to_csv(data_dir + "/subregion_menus.csv")
        region_deals_frame.to_csv(data_dir + "/region_deals.csv")

        # Save to JSON
        self.save_json(export_file, self.data_lib)
        print(f"Time: {datetime.now() - start_time}")

if __name__ == "__main__":
    app = Snooper()
    app.main()


from datetime import datetime
from datetime import timedelta
from config import app

from models.shift import Shift
from utils import date_convertor, shifts_db

class Employee:
    def __init__(self, employee_no):
        self.employee_no = employee_no
        self.first_name = app.redis.get('first_name')
        self.last_name = app.redis.get('last_name')
        self.birthdate = app.redis.get('birthdate')
        self.gender = app.redis.get('gender')
        self.hire_date = app.redis.get('hire_date')
        self.title = app.redis.get('title')
        self.from_date = app.redis.get('from_date')

    def start_shift(self, clock_in_time_str, punch_type):
        shifts = shifts_db.get_shifts_from_db_to_list(self.employee_no)
        clock_in_time_dtime = date_convertor.convert_string_to_datetime(clock_in_time_str, 'datetime')
        response = {'error': True, 'message': f'shift cannot start at {clock_in_time_str}'}
        # loop thru each shift and find the current one
        for shift in shifts:
            if shift.check_if_is_current_shift(clock_in_time_dtime, punch_type):
                print('is current')
                result = shift.set_actual_start_time(clock_in_time_dtime)
                response['error'] = result[0]
                response['message'] = result[1]
        return response

    def end_shift(self, clock_out_time_str, punch_type):
        shifts = shifts_db.get_shifts_from_db_to_list(self.employee_no)
        clock_out_time_dtime = date_convertor.convert_string_to_datetime(clock_out_time_str, 'datetime')
        response = {'error': True, 'message': f'shift cannot end at {clock_out_time_str}'}
        # loop thru each shift and find the current one
        for shift in shifts:
            if shift.check_if_is_current_shift(clock_out_time_dtime, punch_type):
                result = shift.set_actual_end_time(clock_out_time_dtime)
                response['error'] = result[0]
                response['message'] = result[1]
        return response

    def start_lunch(self, lunch_start_time_str, punch_type):
        shifts = shifts_db.get_shifts_from_db_to_list(self.employee_no)
        lunch_start_time_dtime = date_convertor.convert_string_to_datetime(lunch_start_time_str, 'datetime')
        response = {'error': True, 'message': f'lunch cannot start at {lunch_start_time_str}'}
        # loop thru each shift and find the current one
        for shift in shifts:
            if shift.check_if_is_current_shift(lunch_start_time_dtime, punch_type):
                result = shift.set_actual_lunch_start_time(lunch_start_time_dtime)
                response['error'] = result[0]
                response['message'] = result[1]
        return response

    def end_lunch(self, lunch_end_time_str, punch_type):
        shifts = shifts_db.get_shifts_from_db_to_list(self.employee_no)
        lunch_end_time_dtime = date_convertor.convert_string_to_datetime(lunch_end_time_str, 'datetime')
        response = {'error': True, 'message': f'lunch cannot end at {lunch_end_time_str}'}
        # loop thru each shift and find the current one
        for shift in shifts:
            if shift.check_if_is_current_shift(lunch_end_time_dtime, punch_type):
                result = shift.set_actual_lunch_end_time(lunch_end_time_dtime)
                response['error'] = result[0]
                response['message'] = result[1]
        return response

    def update_profile(self, motto, profile_pic_url):
        conn = app.mysql.connection
        cur = conn.cursor()
        try:
            cur.execute("UPDATE employees SET motto = %s, profile_pic = %s WHERE emp_no = %s",
                        (motto, profile_pic_url, self.employee_no))
            conn.commit()
            return {'error': False, 'message': 'Profile updated successfully'}
        except Exception as e:
            conn.rollback()
            return {'error': True, 'message': str(e)}
        finally:
            cur.close()


"""Module logs for customized logger creation.

Author: Jan Jakub Kubik (xkubik32)
Date: 12.3.2023
"""

import logging
from typing import Callable, Union

import structlog

def _create(
    name: str, get_logger_func: Callable, logging_options=logging
) -> Union[logging.Logger, structlog.BoundLogger]:
    """Create a logger with the given name.

    Args:
        name (str): Name of the logger.
        get_logger_func (Callable): Function for creating logger object.
        logging_options (Optional): Logging options. Default is Python's logging module.

    Returns:
        Union[logging.Logger, structlog.BoundLogger]: Logger object based on get_logger_func.
    """
    logging.setLoggerClass(logging_options.Logger)
    logger = get_logger_func(name)

    return logger

def create_logger(name: str) -> structlog.BoundLogger:
    """Create a logger with the given name.

    Create a structlog logger object with the given name.

    Args:
        name (str): Name of the logger.

    Returns:
        structlog.BoundLogger: Structlog logger object with the given name.
    """
    # Same as default processors, just without timestamps
    structlog.configure(
        processors=[
            structlog.processors.StackInfoRenderer(),
            structlog.processors.format_exc_info,
            # structlog.processors.TimeStamper(fmt="%Y-%m-%d %H:%M:%S", utc=True),
            structlog.dev.set_exc_info,
            structlog.dev.ConsoleRenderer(colors=True),
        ],
        wrapper_class=structlog.BoundLogger,
    )
    return _create(name, structlog.getLogger)


"""
@File : loss.py
@Author : CodeCat
@Time : 2021/7/28 上午9:57
"""
import torch
import torch.nn as nn
import torch.nn.functional as F

# 二分类
class FocalLoss(nn.Module):

    def __init__(self, gamma=2, alpha=0.5):
        super(FocalLoss, self).__init__()
        self.gamma = gamma
        self.alpha = alpha

    def forward(self, input, target):
        # input:size is M*2. M　is the batch　number
        # target:size is M.
        pt = torch.softmax(input, dim=1)
        p = pt[:, 1]
        loss = -self.alpha * (1 - p) ** self.gamma * (target * torch.log(p)) - (1 - self.alpha) * p ** self.gamma * ((1 - target) * torch.log(1 - p))
        return loss.mean()

class CEWeightLoss(nn.Module):
    def __init__(self, reduction: str = 'mean'):
        super(CEWeightLoss, self).__init__()
        self.reduction = reduction

    def forward(self, input, target):
        # input:size is M*2. M　is the batch　number
        # target:size is M.
        weight = torch.sum(target) / target.numel()
        weights = torch.Tensor([weight, 1-weight]).to(input.device)
        return F.cross_entropy(input, target, weight=weights, reduction=self.reduction)

class DiceLoss(nn.Module):
    def __init__(self, eps=1e-6):
        super(DiceLoss, self).__init__()
        self.eps = eps

    def forward(self, input, target):
        # input:size is M*2. M　is the batch　number
        # target:size is M.
        neg = 1 - target
        pos = target
        target = torch.stack((neg, pos), dim=1)
        intersection = torch.sum(input * target)
        union = torch.sum(input) + torch.sum(target) + self.eps
        loss = 1 - 2.0 * intersection / union
        return loss


from django.urls import reverse

from core.tests.base import UserBaseTestCase

class SimpleViewsTestCase(UserBaseTestCase):

    def test_renders_main_view_redirect_code(self):
        response = self.client.get(reverse('main'))
        self.assertEqual(302, response.status_code)

    def test_renders_main_view_redirected(self):
        with self.assertTemplateUsed('referrals/index_referrals.html'):
            response = self.client.get(reverse('main'),
                                       follow=True)
            last_url, last_status_code = \
                response.redirect_chain[-1]
            self.assertEquals(last_url, '/en/referrals/')
            self.assertEquals(last_status_code, 302)
            self.assertEqual(200, response.status_code)


import argparse
import logging
import multiprocessing as mp
import os
import os.path
import random
import signal
import sys
import time
from copy import deepcopy

import cherrypy
import cherrypy_cors
from cherrypy.lib import auth_digest

from sflib import SpiderFoot
from sfscan import startSpiderFootScanner
from sfwebui import SpiderFootWebUi
from spiderfoot import SpiderFootHelpers
from spiderfoot import SpiderFootDb
from spiderfoot import SpiderFootCorrelator
from spiderfoot.logger import logListenerSetup, logWorkerSetup
from spiderfoot import __version__

scanId = None
dbh = None

def main() -> None:
    # web server config
    sfWebUiConfig = {
        'host': '127.0.0.1',
        'port': 5001,
        'root': '/',
        'cors_origins': [],
    }

    # 'Global' configuration options
    # These can be overriden on a per-module basis, and some will
    # be overridden from saved configuration settings stored in the DB.
    sfConfig = {
        '_debug': False,  # Debug
        '_maxthreads': 3,  # Number of modules to run concurrently
        '__logging': True,  # Logging in general
        '__outputfilter': None,  # Event types to filter from modules' output
        '_useragent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0',  # User-Agent to use for HTTP requests
        '_dnsserver': '',  # Override the default resolver
        '_fetchtimeout': 5,  # number of seconds before giving up on a fetch
        '_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat',
        '_internettlds_cache': 72,
        '_genericusers': ",".join(SpiderFootHelpers.usernamesFromWordlists(['generic-usernames'])),
        '__database': f"{SpiderFootHelpers.dataPath()}/spiderfoot.db",
        '__modules__': None,  # List of modules. Will be set after start-up.
        '__correlationrules__': None,  # List of correlation rules. Will be set after start-up.
        '_socks1type': '',
        '_socks2addr': '',
        '_socks3port': '',
        '_socks4user': '',
        '_socks5pwd': '',
    }

    sfOptdescs = {
        '_debug': "Enable debugging?",
        '_maxthreads': "Max number of modules to run concurrently",
        '_useragent': "User-Agent string to use for HTTP requests. Prefix with an '@' to randomly select the User Agent from a file containing user agent strings for each request, e.g. @C:\\useragents.txt or @/home/bob/useragents.txt. Or supply a URL to load the list from there.",
        '_dnsserver': "Override the default resolver with another DNS server. For example, 8.8.8.8 is Google's open DNS server.",
        '_fetchtimeout': "Number of seconds before giving up on a HTTP request.",
        '_internettlds': "List of Internet TLDs.",
        '_internettlds_cache': "Hours to cache the Internet TLD list. This can safely be quite a long time given that the list doesn't change too often.",
        '_genericusers': "List of usernames that if found as usernames or as part of e-mail addresses, should be treated differently to non-generics.",
        '_socks1type': "SOCKS Server Type. Can be '4', '5', 'HTTP' or 'TOR'",
        '_socks2addr': 'SOCKS Server IP Address.',
        '_socks3port': 'SOCKS Server TCP Port. Usually 1080 for 4/5, 8080 for HTTP and 9050 for TOR.',
        '_socks4user': 'SOCKS Username. Valid only for SOCKS4 and SOCKS5 servers.',
        '_socks5pwd': "SOCKS Password. Valid only for SOCKS5 servers.",
        '_modulesenabled': "Modules enabled for the scan."  # This is a hack to get a description for an option not actually available.
    }

    # Legacy way to run the server
    args = None
    p = argparse.ArgumentParser(description=f"SpiderFoot {__version__}: Open Source Intelligence Automation.")
    p.add_argument("-d", "--debug", action='store_true', help="Enable debug output.")
    p.add_argument("-l", metavar="IP:port", help="IP and port to listen on.")
    p.add_argument("-m", metavar="mod1,mod2,...", type=str, help="Modules to enable.")
    p.add_argument("-M", "--modules", action='store_true', help="List available modules.")
    p.add_argument("-C", "--correlate", metavar="scanID", help="Run correlation rules against a scan ID.")
    p.add_argument("-s", metavar="TARGET", help="Target for the scan.")
    p.add_argument("-t", metavar="type1,type2,...", type=str, help="Event types to collect (modules selected automatically).")
    p.add_argument("-u", choices=["all", "footprint", "investigate", "passive"], type=str, help="Select modules automatically by use case")
    p.add_argument("-T", "--types", action='store_true', help="List available event types.")
    p.add_argument("-o", choices=["tab", "csv", "json"], type=str, help="Output format. Tab is default.")
    p.add_argument("-H", action='store_true', help="Don't print field headers, just data.")
    p.add_argument("-n", action='store_true', help="Strip newlines from data.")
    p.add_argument("-r", action='store_true', help="Include the source data field in tab/csv output.")
    p.add_argument("-S", metavar="LENGTH", type=int, help="Maximum data length to display. By default, all data is shown.")
    p.add_argument("-D", metavar='DELIMITER', type=str, help="Delimiter to use for CSV output. Default is ,.")
    p.add_argument("-f", action='store_true', help="Filter out other event types that weren't requested with -t.")
    p.add_argument("-F", metavar="type1,type2,...", type=str, help="Show only a set of event types, comma-separated.")
    p.add_argument("-x", action='store_true', help="STRICT MODE. Will only enable modules that can directly consume your target, and if -t was specified only those events will be consumed by modules. This overrides -t and -m options.")
    p.add_argument("-q", action='store_true', help="Disable logging. This will also hide errors!")
    p.add_argument("-V", "--version", action='store_true', help="Display the version of SpiderFoot and exit.")
    p.add_argument("-max-threads", type=int, help="Max number of modules to run concurrently.")
    args = p.parse_args()

    if args.version:
        print(f"SpiderFoot {__version__}: Open Source Intelligence Automation.")
        sys.exit(0)

    if args.max_threads:
        sfConfig['_maxthreads'] = args.max_threads

    if args.debug:
        sfConfig['_debug'] = True
    else:
        sfConfig['_debug'] = False

    if args.q:
        sfConfig['__logging'] = False

    loggingQueue = mp.Queue()
    logListenerSetup(loggingQueue, sfConfig)
    logWorkerSetup(loggingQueue)
    log = logging.getLogger(f"spiderfoot.{__name__}")

    # Add descriptions of the global config options
    sfConfig['__globaloptdescs__'] = sfOptdescs

    # Load each module in the modules directory with a .py extension
    try:
        mod_dir = os.path.dirname(os.path.abspath(__file__)) + '/modules/'
        sfModules = SpiderFootHelpers.loadModulesAsDict(mod_dir, ['sfp_template.py'])
    except BaseException as e:
        log.critical(f"Failed to load modules: {e}", exc_info=True)
        sys.exit(-1)

    if not sfModules:
        log.critical(f"No modules found in modules directory: {mod_dir}")
        sys.exit(-1)

    # Load each correlation rule in the correlations directory with
    # a .yaml extension
    try:
        correlations_dir = os.path.dirname(os.path.abspath(__file__)) + '/correlations/'
        correlationRulesRaw = SpiderFootHelpers.loadCorrelationRulesRaw(correlations_dir, ['template.yaml'])
    except BaseException as e:
        log.critical(f"Failed to load correlation rules: {e}", exc_info=True)
        sys.exit(-1)

    # Initialize database handle
    try:
        dbh = SpiderFootDb(sfConfig)
    except Exception as e:
        log.critical(f"Failed to initialize database: {e}", exc_info=True)
        sys.exit(-1)

    # Sanity-check the rules and parse them
    sfCorrelationRules = list()
    if not correlationRulesRaw:
        log.error(f"No correlation rules found in correlations directory: {correlations_dir}")
    else:
        try:
            correlator = SpiderFootCorrelator(dbh, correlationRulesRaw)
            sfCorrelationRules = correlator.get_ruleset()
        except Exception as e:
            log.critical(f"Failure initializing correlation rules: {e}", exc_info=True)
            sys.exit(-1)

    # Add modules and correlation rules to sfConfig so they can be used elsewhere
    sfConfig['__modules__'] = sfModules
    sfConfig['__correlationrules__'] = sfCorrelationRules

    if args.correlate:
        if not correlationRulesRaw:
            log.error("Unable to perform correlations as no correlation rules were found.")
            sys.exit(-1)

        try:
            log.info(f"Running {len(correlationRulesRaw)} correlation rules against scan, {args.correlate}.")
            corr = SpiderFootCorrelator(dbh, correlationRulesRaw, args.correlate)
            corr.run_correlations()
        except Exception as e:
            log.critical(f"Unable to run correlation rules: {e}", exc_info=True)
            sys.exit(-1)
        sys.exit(0)

    if args.modules:
        log.info("Modules available:")
        for m in sorted(sfModules.keys()):
            if "__" in m:
                continue
            print(f"{m.ljust(25)}  {sfModules[m]['descr']}")
        sys.exit(0)

    if args.types:
        dbh = SpiderFootDb(sfConfig, init=True)
        log.info("Types available:")
        typedata = dbh.eventTypes()
        types = dict()
        for r in typedata:
            types[r[1]] = r[0]

        for t in sorted(types.keys()):
            print(f"{t.ljust(45)}  {types[t]}")
        sys.exit(0)

    if args.l:
        try:
            (host, port) = args.l.split(":")
        except BaseException:
            log.critical("Invalid ip:port format.")
            sys.exit(-1)

        sfWebUiConfig['host'] = host
        sfWebUiConfig['port'] = port

        start_web_server(sfWebUiConfig, sfConfig, loggingQueue)
        sys.exit(0)

    start_scan(sfConfig, sfModules, args, loggingQueue)

def start_scan(sfConfig: dict, sfModules: dict, args, loggingQueue) -> None:
    """Start scan

    Args:
        sfConfig (dict): SpiderFoot config options
        sfModules (dict): modules
        args (argparse.Namespace): command line args
        loggingQueue (Queue): main SpiderFoot logging queue
    """
    log = logging.getLogger(f"spiderfoot.{__name__}")

    global dbh
    global scanId

    dbh = SpiderFootDb(sfConfig, init=True)
    sf = SpiderFoot(sfConfig)

    if not args.s:
        log.error("You must specify a target when running in scan mode. Try --help for guidance.")
        sys.exit(-1)

    if args.x and not args.t:
        log.error("-x can only be used with -t. Use --help for guidance.")
        sys.exit(-1)

    if args.x and args.m:
        log.error("-x can only be used with -t and not with -m. Use --help for guidance.")
        sys.exit(-1)

    if args.r and (args.o and args.o not in ["tab", "csv"]):
        log.error("-r can only be used when your output format is tab or csv.")
        sys.exit(-1)

    if args.H and (args.o and args.o not in ["tab", "csv"]):
        log.error("-H can only be used when your output format is tab or csv.")
        sys.exit(-1)

    if args.D and args.o != "csv":
        log.error("-D can only be used when using the csv output format.")
        sys.exit(-1)

    target = args.s
    # Usernames and names - quoted on the commandline - won't have quotes,
    # so add them.
    if " " in target:
        target = f"\"{target}\""
    if "." not in target and not target.startswith("+") and '"' not in target:
        target = f"\"{target}\""
    targetType = SpiderFootHelpers.targetTypeFromString(target)

    if not targetType:
        log.error(f"Could not determine target type. Invalid target: {target}")
        sys.exit(-1)

    target = target.strip('"')

    modlist = list()
    if not args.t and not args.m and not args.u:
        log.warning("You didn't specify any modules, types or use case, so all modules will be enabled.")
        for m in list(sfModules.keys()):
            if "__" in m:
                continue
            modlist.append(m)

    signal.signal(signal.SIGINT, handle_abort)
    # If the user is scanning by type..
    # 1. Find modules producing that type
    if args.t:
        types = args.t
        modlist = sf.modulesProducing(types)
        newmods = deepcopy(modlist)
        newmodcpy = deepcopy(newmods)

        # 2. For each type those modules consume, get modules producing
        while len(newmodcpy) > 0:
            for etype in sf.eventsToModules(newmodcpy):
                xmods = sf.modulesProducing([etype])
                for mod in xmods:
                    if mod not in modlist:
                        modlist.append(mod)
                        newmods.append(mod)
            newmodcpy = deepcopy(newmods)
            newmods = list()

    # Easier if scanning by module
    if args.m:
        modlist = list(filter(None, args.m.split(",")))

    # Select modules if the user selected usercase
    if args.u:
        usecase = args.u[0].upper() + args.u[1:]  # Make the first Letter Uppercase
        for mod in sfConfig['__modules__']:
            if usecase == 'All' or usecase in sfConfig['__modules__'][mod]['group']:
                modlist.append(mod)

    # Add sfp__stor_stdout to the module list
    typedata = dbh.eventTypes()
    types = dict()
    for r in typedata:
        types[r[1]] = r[0]

    sfp__stor_stdout_opts = sfConfig['__modules__']['sfp__stor_stdout']['opts']
    sfp__stor_stdout_opts['_eventtypes'] = types
    if args.f:
        if args.f and not args.t:
            log.error("You can only use -f with -t. Use --help for guidance.")
            sys.exit(-1)
        sfp__stor_stdout_opts['_showonlyrequested'] = True
    if args.F:
        sfp__stor_stdout_opts['_requested'] = args.F.split(",")
        sfp__stor_stdout_opts['_showonlyrequested'] = True
    if args.o:
        if args.o not in ["tab", "csv", "json"]:
            log.error("Invalid output format selected. Must be 'tab', 'csv' or 'json'.")
            sys.exit(-1)
        sfp__stor_stdout_opts['_format'] = args.o
    if args.t:
        sfp__stor_stdout_opts['_requested'] = args.t.split(",")
    if args.n:
        sfp__stor_stdout_opts['_stripnewline'] = True
    if args.r:
        sfp__stor_stdout_opts['_showsource'] = True
    if args.S:
        sfp__stor_stdout_opts['_maxlength'] = args.S
    if args.D:
        sfp__stor_stdout_opts['_csvdelim'] = args.D
    if args.x:
        tmodlist = list()
        modlist = list()
        xmods = sf.modulesConsuming([targetType])
        for mod in xmods:
            if mod not in modlist:
                tmodlist.append(mod)

        # Remove any modules not producing the type requested
        rtypes = args.t.split(",")
        for mod in tmodlist:
            for r in rtypes:
                if not sfModules[mod]['provides']:
                    continue
                if r in sfModules[mod].get('provides', []) and mod not in modlist:
                    modlist.append(mod)

    if len(modlist) == 0:
        log.error("Based on your criteria, no modules were enabled.")
        sys.exit(-1)

    modlist += ["sfp__stor_db", "sfp__stor_stdout"]

    if sfConfig['__logging']:
        log.info(f"Modules enabled ({len(modlist)}): {','.join(modlist)}")

    cfg = sf.configUnserialize(dbh.configGet(), sfConfig)

    # Debug mode is a variable that gets stored to the DB, so re-apply it
    if args.debug:
        cfg['_debug'] = True
    else:
        cfg['_debug'] = False

    # If strict mode is enabled, filter the output from modules.
    if args.x and args.t:
        cfg['__outputfilter'] = args.t.split(",")

    # Prepare scan output headers
    if args.o == "json":
        print("[", end='')
    elif not args.H:
        delim = "\t"

        if args.o == "tab":
            delim = "\t"

        if args.o == "csv":
            if args.D:
                delim = args.D
            else:
                delim = ","

        if args.r:
            if delim == "\t":
                headers = delim.join(["Source".ljust(30), "Type".ljust(45), "Source Data", "Data"])
            else:
                headers = delim.join(["Source", "Type", "Source Data", "Data"])
        else:
            if delim == "\t":
                headers = delim.join(["Source".ljust(30), "Type".ljust(45), "Data"])
            else:
                headers = delim.join(["Source", "Type", "Data"])

        print(headers)

    # Start running a new scan
    scanName = target
    scanId = SpiderFootHelpers.genScanInstanceId()
    try:
        p = mp.Process(target=startSpiderFootScanner, args=(loggingQueue, scanName, scanId, target, targetType, modlist, cfg))
        p.daemon = True
        p.start()
    except BaseException as e:
        log.error(f"Scan [{scanId}] failed: {e}")
        sys.exit(-1)

    # Poll for scan status until completion
    while True:
        time.sleep(1)
        info = dbh.scanInstanceGet(scanId)
        if not info:
            continue
        if info[5] in ["ERROR-FAILED", "ABORT-REQUESTED", "ABORTED", "FINISHED"]:
            if sfConfig['__logging']:
                log.info(f"Scan completed with status {info[5]}")
            if args.o == "json":
                print("]")
            sys.exit(0)

    return

def start_web_server(sfWebUiConfig: dict, sfConfig: dict, loggingQueue=None) -> None:
    """Start the web server so you can start looking at results

    Args:
        sfWebUiConfig (dict): web server options
        sfConfig (dict): SpiderFoot config options
        loggingQueue (Queue): main SpiderFoot logging queue
    """
    log = logging.getLogger(f"spiderfoot.{__name__}")

    web_host = sfWebUiConfig.get('host', '127.0.0.1')
    web_port = sfWebUiConfig.get('port', 5001)
    web_root = sfWebUiConfig.get('root', '/')
    cors_origins = sfWebUiConfig.get('cors_origins', [])

    cherrypy.config.update({
        'log.screen': False,
        'server.socket_host': web_host,
        'server.socket_port': int(web_port)
    })

    log.info(f"Starting web server at {web_host}:{web_port} ...")

    # Enable access to static files via the web directory
    conf = {
        '/query': {
            'tools.encode.text_only': False,
            'tools.encode.add_charset': True,
        },
        '/static': {
            'tools.staticdir.on': True,
            'tools.staticdir.dir': 'static',
            'tools.staticdir.root': f"{os.path.dirname(os.path.abspath(__file__))}/spiderfoot"
        }
    }

    secrets = dict()
    passwd_file = SpiderFootHelpers.dataPath() + '/passwd'
    if os.path.isfile(passwd_file):
        if not os.access(passwd_file, os.R_OK):
            log.error("Could not read passwd file. Permission denied.")
            sys.exit(-1)

        with open(passwd_file, 'r') as f:
            passwd_data = f.readlines()

        for line in passwd_data:
            if line.strip() == '':
                continue

            if ':' not in line:
                log.error("Incorrect format of passwd file, must be username:password on each line.")
                sys.exit(-1)

            u = line.strip().split(":")[0]
            p = ':'.join(line.strip().split(":")[1:])

            if not u or not p:
                log.error("Incorrect format of passwd file, must be username:password on each line.")
                sys.exit(-1)

            secrets[u] = p

    if secrets:
        log.info("Enabling authentication based on supplied passwd file.")
        conf['/'] = {
            'tools.auth_digest.on': True,
            'tools.auth_digest.realm': web_host,
            'tools.auth_digest.get_ha1': auth_digest.get_ha1_dict_plain(secrets),
            'tools.auth_digest.key': random.SystemRandom().randint(0, 99999999)
        }
    else:
        warn_msg = "\n********************************************************************\n"
        warn_msg += "Warning: passwd file contains no passwords. Authentication disabled.\n"
        warn_msg += "Please consider adding authentication to protect this instance!\n"
        warn_msg += "Refer to https://www.spiderfoot.net/documentation/#security.\n"
        warn_msg += "********************************************************************\n"
        log.warning(warn_msg)

    using_ssl = False
    key_path = SpiderFootHelpers.dataPath() + '/spiderfoot.key'
    crt_path = SpiderFootHelpers.dataPath() + '/spiderfoot.crt'
    if os.path.isfile(key_path) and os.path.isfile(crt_path):
        if not os.access(crt_path, os.R_OK):
            log.critical(f"Could not read {crt_path} file. Permission denied.")
            sys.exit(-1)

        if not os.access(key_path, os.R_OK):
            log.critical(f"Could not read {key_path} file. Permission denied.")
            sys.exit(-1)

        log.info("Enabling SSL based on supplied key and certificate file.")
        cherrypy.server.ssl_module = 'builtin'
        cherrypy.server.ssl_certificate = crt_path
        cherrypy.server.ssl_private_key = key_path
        using_ssl = True

    if using_ssl:
        url = "https://"
    else:
        url = "http://"

    if web_host == "0.0.0.0":  # nosec
        url = f"{url}127.0.0.1:{web_port}"
    else:
        url = f"{url}{web_host}:{web_port}{web_root}"
        cors_origins.append(url)

    cherrypy_cors.install()
    cherrypy.config.update({
        'cors.expose.on': True,
        'cors.expose.origins': cors_origins,
        'cors.preflight.origins': cors_origins
    })

    print("")
    print("*************************************************************")
    print(" Use SpiderFoot by starting your web browser of choice and ")
    print(f" browse to {url}")
    print("*************************************************************")
    print("")

    # Disable auto-reloading of content
    cherrypy.engine.autoreload.unsubscribe()

    cherrypy.quickstart(SpiderFootWebUi(sfWebUiConfig, sfConfig, loggingQueue), script_name=web_root, config=conf)

def handle_abort(signal, frame) -> None:
    """Handle interrupt and abort scan.

    Args:
        signal: TBD
        frame: TBD
    """
    log = logging.getLogger(f"spiderfoot.{__name__}")

    global dbh
    global scanId

    if scanId and dbh:
        log.info(f"Aborting scan [{scanId}] ...")
        dbh.scanInstanceSet(scanId, None, None, "ABORTED")
    sys.exit(-1)

if __name__ == '__main__':
    if sys.version_info < (3, 7):
        print("SpiderFoot requires Python 3.7 or higher.")
        sys.exit(-1)

    if len(sys.argv) <= 1:
        print("SpiderFoot requires -l <ip>:<port> to start the web server. Try --help for guidance.")
        sys.exit(-1)

    # TODO: remove this after a few releases (added in 3.5 pre-release 2021-09-05)
    from pathlib import Path
    if os.path.exists('spiderfoot.db'):
        print(f"ERROR: spiderfoot.db file exists in {os.path.dirname(__file__)}")
        print("SpiderFoot no longer supports loading the spiderfoot.db database from the application directory.")
        print(f"The database is now loaded from your home directory: {Path.home()}/.spiderfoot/spiderfoot.db")
        print(f"This message will go away once you move or remove spiderfoot.db from {os.path.dirname(__file__)}")
        sys.exit(-1)

    # TODO: remove this after a few releases (added in 3.5 pre-release 2021-09-05)
    from pathlib import Path
    if os.path.exists('passwd'):
        print(f"ERROR: passwd file exists in {os.path.dirname(__file__)}")
        print("SpiderFoot no longer supports loading credentials from the application directory.")
        print(f"The passwd file is now loaded from your home directory: {Path.home()}/.spiderfoot/passwd")
        print(f"This message will go away once you move or remove passwd from {os.path.dirname(__file__)}")
        sys.exit(-1)

    main()