from typing import cast
from unittest.mock import AsyncMock, patch

import pytest

from starknet_py.contract import Contract
from starknet_py.hash.address import compute_address
from starknet_py.hash.selector import get_selector_from_name
from starknet_py.net.account.account import Account
from starknet_py.net.account.base_account import BaseAccount
from starknet_py.net.client_errors import ClientError
from starknet_py.net.client_models import (
    Call,
    DeployAccountTransaction,
    DeployAccountTransactionResponse,
    EstimatedFee,
    SierraContractClass,
    TransactionStatus,
)
from starknet_py.net.full_node_client import FullNodeClient
from starknet_py.net.gateway_client import GatewayClient
from starknet_py.net.models import StarknetChainId
from starknet_py.net.models.transaction import Declare, DeclareV2
from starknet_py.net.signer.stark_curve_signer import KeyPair
from starknet_py.net.udc_deployer.deployer import Deployer
from starknet_py.tests.e2e.fixtures.constants import MAX_FEE

@pytest.mark.run_on_devnet
@pytest.mark.asyncio
@pytest.mark.parametrize(
    "client",
    [GatewayClient(net="custom.net"), FullNodeClient(node_url="custom.net/rpc")],
)
async def test_get_balance_throws_when_token_not_specified(account, client):
    modified_account = Account(
        address=account.address,
        client=client,
        key_pair=KeyPair(1, 2),
        chain=cast(StarknetChainId, 1),
    )
    with pytest.raises(
        ValueError,
        match="Argument token_address must be specified when using a custom network.",
    ):
        await modified_account.get_balance()

@pytest.mark.asyncio
async def test_balance_when_token_specified(account, erc20_contract):
    balance = await account.get_balance(erc20_contract.address)

    assert balance == 200

@pytest.mark.asyncio
async def test_estimated_fee_greater_than_zero(erc20_contract, account):
    erc20_contract = Contract(
        address=erc20_contract.address, abi=erc20_contract.data.abi, provider=account
    )

    estimated_fee = (
        await erc20_contract.functions["balanceOf"]
        .prepare("1234", max_fee=0)
        .estimate_fee(block_hash="latest")
    )

    assert estimated_fee.overall_fee > 0
    assert (
        estimated_fee.gas_price * estimated_fee.gas_usage == estimated_fee.overall_fee
    )

@pytest.mark.asyncio
async def test_estimate_fee_for_declare_transaction(account, map_compiled_contract):
    declare_tx = await account.sign_declare_transaction(
        compiled_contract=map_compiled_contract, max_fee=MAX_FEE
    )

    estimated_fee = await account.client.estimate_fee(tx=declare_tx)

    assert isinstance(estimated_fee.overall_fee, int)
    assert estimated_fee.overall_fee > 0
    assert (
        estimated_fee.gas_usage * estimated_fee.gas_price == estimated_fee.overall_fee
    )

@pytest.mark.asyncio
@pytest.mark.parametrize("key, val", [(20, 20), (30, 30)])
async def test_sending_multicall(account, map_contract, key, val):
    calls = [
        map_contract.functions["put"].prepare(key=10, value=10),
        map_contract.functions["put"].prepare(key=key, value=val),
    ]

    res = await account.execute(calls=calls, max_fee=int(1e20))
    await account.client.wait_for_tx(res.transaction_hash)

    (value,) = await map_contract.functions["get"].call(key=key)

    assert value == val

@pytest.mark.run_on_devnet
@pytest.mark.asyncio
async def test_get_block_traces(gateway_account):
    traces = await gateway_account.client.trace_block_transactions(block_number=2)

    assert traces.traces != []

@pytest.mark.asyncio
async def test_rejection_reason_in_transaction_receipt(map_contract):
    with pytest.raises(ClientError, match=r".*INSUFFICIENT_MAX_FEE.*"):
        _ = await map_contract.functions["put"].invoke(key=10, value=20, max_fee=1)

def test_sign_and_verify_offchain_message_fail(account, typed_data):
    signature = account.sign_message(typed_data)
    signature = [signature[0] + 1, signature[1]]
    result = account.verify_message(typed_data, signature)

    assert result is False

def test_sign_and_verify_offchain_message(account, typed_data):
    signature = account.sign_message(typed_data)
    result = account.verify_message(typed_data, signature)

    assert result is True

@pytest.mark.asyncio
async def test_get_class_hash_at(map_contract, account):
    class_hash = await account.client.get_class_hash_at(
        map_contract.address, block_hash="latest"
    )

    assert class_hash != 0

@pytest.mark.asyncio()
async def test_get_nonce(account, map_contract):
    nonce = await account.get_nonce()
    address = map_contract.address
    block = await account.client.get_block(block_number="latest")

    tx = await account.execute(
        Call(
            to_addr=address, selector=get_selector_from_name("put"), calldata=[10, 20]
        ),
        max_fee=MAX_FEE,
    )
    await account.client.wait_for_tx(tx.transaction_hash)

    new_nonce = await account.get_nonce()
    new_nonce_latest_block = await account.get_nonce(block_number="latest")

    old_nonce = await account.get_nonce(block_number=block.block_number)

    assert isinstance(nonce, int) and isinstance(new_nonce, int)
    assert new_nonce == nonce + 1

    assert old_nonce == nonce
    assert new_nonce_latest_block == new_nonce

@pytest.mark.asyncio
@pytest.mark.parametrize(
    "calls", [[Call(10, 20, [30])], [Call(10, 20, [30]), Call(40, 50, [60])]]
)
async def test_sign_invoke_transaction(account, calls):
    signed_tx = await account.sign_invoke_transaction(calls, max_fee=MAX_FEE)

    assert isinstance(signed_tx.signature, list)
    assert len(signed_tx.signature) > 0
    assert signed_tx.max_fee == MAX_FEE

@pytest.mark.asyncio
async def test_sign_invoke_transaction_auto_estimate(account, map_contract):
    signed_tx = await account.sign_invoke_transaction(
        Call(map_contract.address, get_selector_from_name("put"), [3, 4]),
        auto_estimate=True,
    )

    assert isinstance(signed_tx.signature, list)
    assert len(signed_tx.signature) > 0
    assert signed_tx.max_fee > 0

@pytest.mark.asyncio
async def test_sign_declare_transaction(account, map_compiled_contract):
    signed_tx = await account.sign_declare_transaction(
        map_compiled_contract, max_fee=MAX_FEE
    )

    assert isinstance(signed_tx, Declare)
    assert signed_tx.version == 1
    assert isinstance(signed_tx.signature, list)
    assert len(signed_tx.signature) > 0
    assert signed_tx.max_fee == MAX_FEE

@pytest.mark.asyncio
async def test_sign_declare_transaction_auto_estimate(account, map_compiled_contract):
    signed_tx = await account.sign_declare_transaction(
        map_compiled_contract, auto_estimate=True
    )

    assert isinstance(signed_tx, Declare)
    assert signed_tx.version == 1
    assert isinstance(signed_tx.signature, list)
    assert len(signed_tx.signature) > 0
    assert signed_tx.max_fee > 0

@pytest.mark.asyncio
async def test_sign_declare_v2_transaction(
    account, sierra_minimal_compiled_contract_and_class_hash
):
    (
        compiled_contract,
        compiled_class_hash,
    ) = sierra_minimal_compiled_contract_and_class_hash

    signed_tx = await account.sign_declare_v2_transaction(
        compiled_contract,
        compiled_class_hash=compiled_class_hash,
        max_fee=MAX_FEE,
    )

    assert isinstance(signed_tx, DeclareV2)
    assert signed_tx.version == 2
    assert isinstance(signed_tx.signature, list)
    assert len(signed_tx.signature) > 0
    assert signed_tx.max_fee == MAX_FEE

@pytest.mark.asyncio
async def test_sign_declare_v2_transaction_auto_estimate(
    account, sierra_minimal_compiled_contract_and_class_hash
):
    (
        compiled_contract,
        compiled_class_hash,
    ) = sierra_minimal_compiled_contract_and_class_hash

    signed_tx = await account.sign_declare_v2_transaction(
        compiled_contract,
        compiled_class_hash=compiled_class_hash,
        auto_estimate=True,
    )

    assert isinstance(signed_tx, DeclareV2)
    assert signed_tx.version == 2
    assert isinstance(signed_tx.signature, list)
    assert len(signed_tx.signature) > 0
    assert signed_tx.max_fee > 0

@pytest.mark.asyncio
async def test_declare_contract_raises_on_sierra_contract_without_compiled_class_hash(
    sierra_minimal_compiled_contract_and_class_hash, account
):
    compiled_contract, _ = sierra_minimal_compiled_contract_and_class_hash
    with pytest.raises(
        ValueError,
        match="Signing sierra contracts requires using `sign_declare_v2_transaction` method.",
    ):
        await account.sign_declare_transaction(compiled_contract=compiled_contract)

@pytest.mark.asyncio
async def test_sign_deploy_account_transaction(account):
    class_hash = 0x1234
    salt = 0x123
    calldata = [1, 2, 3]
    signed_tx = await account.sign_deploy_account_transaction(
        class_hash, salt, calldata, max_fee=MAX_FEE
    )

    assert isinstance(signed_tx.signature, list)
    assert len(signed_tx.signature) > 0
    assert signed_tx.max_fee == MAX_FEE
    assert signed_tx.class_hash == class_hash
    assert signed_tx.contract_address_salt == salt
    assert signed_tx.constructor_calldata == calldata

@pytest.mark.asyncio
async def test_sign_deploy_account_transaction_auto_estimate(
    account, account_with_validate_deploy_class_hash
):
    class_hash = account_with_validate_deploy_class_hash
    salt = 0x1234
    calldata = [account.signer.public_key]
    signed_tx = await account.sign_deploy_account_transaction(
        class_hash, salt, calldata, auto_estimate=True
    )

    assert isinstance(signed_tx.signature, list)
    assert len(signed_tx.signature) > 0
    assert signed_tx.max_fee > 0
    assert signed_tx.class_hash == class_hash
    assert signed_tx.contract_address_salt == salt
    assert signed_tx.constructor_calldata == calldata

@pytest.mark.skip("No 0.12.1 devnet")
@pytest.mark.asyncio
async def test_deploy_account(client, deploy_account_details_factory, map_contract):
    address, key_pair, salt, class_hash = await deploy_account_details_factory.get()

    deploy_result = await Account.deploy_account(
        address=address,
        class_hash=class_hash,
        salt=salt,
        key_pair=key_pair,
        client=client,
        chain=StarknetChainId.TESTNET,
        max_fee=int(1e16),
    )
    await deploy_result.wait_for_acceptance()

    account = deploy_result.account

    assert isinstance(account, BaseAccount)
    assert account.address == address

    # Test making a tx
    res = await account.execute(
        calls=Call(
            to_addr=map_contract.address,
            selector=get_selector_from_name("put"),
            calldata=[30, 40],
        ),
        max_fee=MAX_FEE,
    )
    tx_receipt = await account.client.wait_for_tx(res.transaction_hash)

    assert tx_receipt.finality_status in (
        TransactionStatus.ACCEPTED_ON_L1,
        TransactionStatus.ACCEPTED_ON_L2,
    )

@pytest.mark.asyncio
async def test_deploy_account_raises_on_incorrect_address(
    client, deploy_account_details_factory
):
    address, key_pair, salt, class_hash = await deploy_account_details_factory.get()

    with pytest.raises(
        ValueError,
        match=f"Provided address {hex(0x111)} is different than computed address {hex(address)}",
    ):
        await Account.deploy_account(
            address=0x111,
            class_hash=class_hash,
            salt=salt,
            key_pair=key_pair,
            client=client,
            chain=StarknetChainId.TESTNET,
            max_fee=MAX_FEE,
        )

@pytest.mark.asyncio
@pytest.mark.parametrize(
    "call_contract, client",
    [
        (
            "starknet_py.net.gateway_client.GatewayClient.call_contract",
            "gateway_client",
        ),
        (
            "starknet_py.net.full_node_client.FullNodeClient.call_contract",
            "full_node_client",
        ),
    ],
)
async def test_deploy_account_raises_on_no_enough_funds(
    deploy_account_details_factory, call_contract, client, request
):
    client = request.getfixturevalue(client)
    address, key_pair, salt, class_hash = await deploy_account_details_factory.get()

    with patch(call_contract, AsyncMock()) as mocked_balance:
        mocked_balance.return_value = (0, 0)

        with pytest.raises(
            ValueError,
            match="Not enough tokens at the specified address to cover deployment costs",
        ):
            await Account.deploy_account(
                address=address,
                class_hash=class_hash,
                salt=salt,
                key_pair=key_pair,
                client=client,
                chain=StarknetChainId.TESTNET,
                max_fee=MAX_FEE,
            )

@pytest.mark.asyncio
@pytest.mark.parametrize(
    "client_method_path, client",
    [
        (
            "starknet_py.net.gateway_client.GatewayClient",
            "gateway_client",
        ),
        (
            "starknet_py.net.full_node_client.FullNodeClient",
            "full_node_client",
        ),
    ],
)
async def test_deploy_account_passes_on_enough_funds(
    deploy_account_details_factory, client_method_path, client, request
):
    address, key_pair, salt, class_hash = await deploy_account_details_factory.get()
    client = request.getfixturevalue(client)

    with patch(
        client_method_path + ".call_contract", AsyncMock()
    ) as mocked_balance, patch(
        client_method_path + ".deploy_account", AsyncMock()
    ) as mocked_deploy:
        mocked_balance.return_value = (0, 100)
        mocked_deploy.return_value = DeployAccountTransactionResponse(
            transaction_hash=0x1
        )

        await Account.deploy_account(
            address=address,
            class_hash=class_hash,
            salt=salt,
            key_pair=key_pair,
            client=client,
            chain=StarknetChainId.TESTNET,
            max_fee=MAX_FEE,
        )

# TODO (#1056): change this test to braavos account
@pytest.mark.skip(
    reason="'__validate_execute__' doesn't allow any other calldata than in the constructor"
)
@pytest.mark.asyncio
async def test_deploy_account_uses_custom_calldata(
    client, deploy_account_details_factory, fee_contract
):
    _, key_pair, salt, class_hash = await deploy_account_details_factory.get()
    calldata = [1, 2, 3, 4]
    address = compute_address(
        salt=salt,
        class_hash=class_hash,
        constructor_calldata=calldata,
        deployer_address=0,
    )

    res = await fee_contract.functions["transfer"].invoke(
        recipient=address, amount=int(1e16), max_fee=MAX_FEE
    )
    await res.wait_for_acceptance()

    deploy_result = await Account.deploy_account(
        address=address,
        class_hash=class_hash,
        salt=salt,
        key_pair=key_pair,
        client=client,
        chain=StarknetChainId.TESTNET,
        constructor_calldata=calldata,
        max_fee=int(1e16),
    )

    tx = await client.get_transaction(deploy_result.hash)
    assert isinstance(tx, DeployAccountTransaction)
    assert tx.constructor_calldata == calldata

@pytest.mark.asyncio
async def test_sign_invoke_tx_for_fee_estimation(account, map_contract):
    call = map_contract.functions["put"].prepare(key=40, value=50)
    transaction = await account.sign_invoke_transaction(calls=call, max_fee=MAX_FEE)

    estimate_fee_transaction = await account.sign_for_fee_estimate(transaction)

    estimation = await account.client.estimate_fee(estimate_fee_transaction)
    assert estimation.overall_fee > 0

    # Verify that the transaction signed for fee estimation cannot be sent
    with pytest.raises(ClientError):
        await account.client.send_transaction(estimate_fee_transaction)

    # Verify that original transaction can be sent
    result = await account.client.send_transaction(transaction)
    await account.client.wait_for_tx(result.transaction_hash)

@pytest.mark.asyncio
async def test_sign_declare_tx_for_fee_estimation(account, map_compiled_contract):
    transaction = await account.sign_declare_transaction(
        compiled_contract=map_compiled_contract, max_fee=MAX_FEE
    )

    estimate_fee_transaction = await account.sign_for_fee_estimate(transaction)

    estimation = await account.client.estimate_fee(estimate_fee_transaction)
    assert estimation.overall_fee > 0

    # Verify that the transaction signed for fee estimation cannot be sent
    with pytest.raises(ClientError):
        await account.client.declare(estimate_fee_transaction)

    # Verify that original transaction can be sent
    result = await account.client.declare(transaction)
    await account.client.wait_for_tx(result.transaction_hash)

@pytest.mark.asyncio
async def test_sign_deploy_account_tx_for_fee_estimation(
    client, deploy_account_details_factory
):
    address, key_pair, salt, class_hash = await deploy_account_details_factory.get()

    account = Account(
        address=address,
        client=client,
        key_pair=key_pair,
        chain=StarknetChainId.TESTNET,
    )

    transaction = await account.sign_deploy_account_transaction(
        class_hash=class_hash,
        contract_address_salt=salt,
        constructor_calldata=[key_pair.public_key],
        max_fee=MAX_FEE,
    )

    estimate_fee_transaction = await account.sign_for_fee_estimate(transaction)

    estimation = await account.client.estimate_fee(transaction)
    assert isinstance(estimation, EstimatedFee)
    assert estimation.overall_fee > 0

    # Verify that the transaction signed for fee estimation cannot be sent
    with pytest.raises(ClientError):
        await account.client.deploy_account(estimate_fee_transaction)

    # Verify that original transaction can be sent
    result = await account.client.deploy_account(transaction)
    await account.client.wait_for_tx(result.transaction_hash)

@pytest.mark.asyncio
async def test_sign_transaction_custom_nonce(account, cairo1_hello_starknet_class_hash):
    deployment = Deployer().create_contract_deployment(cairo1_hello_starknet_class_hash)
    deploy_tx = await account.sign_invoke_transaction(deployment.call, max_fee=MAX_FEE)

    new_balance = 30
    invoke_tx = await account.sign_invoke_transaction(
        Call(
            deployment.address,
            get_selector_from_name("increase_balance"),
            [new_balance],
        ),
        nonce=deploy_tx.nonce + 1,
        max_fee=MAX_FEE,
    )

    deploy_res = await account.client.send_transaction(deploy_tx)
    invoke_res = await account.client.send_transaction(invoke_tx)

    await account.client.wait_for_tx(deploy_res.transaction_hash)
    await account.client.wait_for_tx(invoke_res.transaction_hash)

    result = await account.client.call_contract(
        Call(deployment.address, get_selector_from_name("get_balance"), [])
    )

    assert invoke_tx.nonce == deploy_tx.nonce + 1
    assert result == [new_balance]

@pytest.mark.asyncio
async def test_argent_cairo1_account_deploy(
    full_node_client,
    argent_cairo1_account_class_hash,
    deploy_account_details_factory,
):
    address, key_pair, salt, class_hash = await deploy_account_details_factory.get(
        class_hash=argent_cairo1_account_class_hash, argent_calldata=True
    )

    deploy_result = await Account.deploy_account(
        address=address,
        class_hash=class_hash,
        salt=salt,
        key_pair=key_pair,
        client=full_node_client,
        constructor_calldata=[key_pair.public_key, 0],
        chain=StarknetChainId.TESTNET,
        max_fee=int(1e16),
    )
    await deploy_result.wait_for_acceptance()
    account = deploy_result.account

    assert isinstance(account, BaseAccount)
    assert await account.cairo_version == 1

    account_contract_class = await full_node_client.get_class_at(
        contract_address=account.address, block_number="latest"
    )

    assert isinstance(account_contract_class, SierraContractClass)

@pytest.mark.asyncio
async def test_argent_cairo1_account_execute(
    deployed_balance_contract,
    argent_cairo1_account: BaseAccount,
):
    # verify that initial balance is 0
    get_balance_call = Call(
        to_addr=deployed_balance_contract.address,
        selector=get_selector_from_name("get_balance"),
        calldata=[],
    )
    get_balance = await argent_cairo1_account.client.call_contract(
        call=get_balance_call, block_number="latest"
    )

    assert get_balance[0] == 0

    value = 20
    increase_balance_by_20_call = Call(
        to_addr=deployed_balance_contract.address,
        selector=get_selector_from_name("increase_balance"),
        calldata=[value],
    )
    execute = await argent_cairo1_account.execute(
        calls=increase_balance_by_20_call, max_fee=int(1e16)
    )
    await argent_cairo1_account.client.wait_for_tx(tx_hash=execute.transaction_hash)
    receipt = await argent_cairo1_account.client.get_transaction_receipt(
        tx_hash=execute.transaction_hash
    )

    # TODO (#1179): devnet 0.3.0 still return STATUS instead of FINALITY_STATUS
    assert receipt.status == TransactionStatus.ACCEPTED_ON_L2

    # verify that the previous call was executed
    get_balance_call = Call(
        to_addr=deployed_balance_contract.address,
        selector=get_selector_from_name("get_balance"),
        calldata=[],
    )
    get_balance = await argent_cairo1_account.client.call_contract(
        call=get_balance_call, block_number="latest"
    )

    assert get_balance[0] == value

# TODO (#1184): remove that
@pytest.mark.asyncio
async def test_cairo1_account_deprecations(
    deployed_balance_contract,
    argent_cairo1_account: BaseAccount,
):
    call = Call(
        to_addr=deployed_balance_contract.address,
        selector=get_selector_from_name("increase_balance"),
        calldata=[20],
    )
    with pytest.warns(
        DeprecationWarning,
        match="Parameter 'cairo_version' has been deprecated. It is calculated automatically based on your account's "
        "contract class.",
    ):
        _ = await argent_cairo1_account.execute(
            calls=call, max_fee=int(1e16), cairo_version=1
        )
        _ = await argent_cairo1_account.sign_invoke_transaction(
            calls=call, max_fee=int(1e16), cairo_version=1
        )


# Aprimore o desafio 093 para que ele funcione com varios jogadores, incluindo um sistema de visualização de detalhes do aproveitamento de cada jogador

lista_time = []
estrutura = {}
lista = []                               # Guarda os gols
while True:
    estrutura.clear()                                       # limpar o dicionario guando começar o laço
    estrutura['nome'] = str(input('Nome do Jogador: ').title())
    quantidade_partidas = int(input(f'Quantas partidas {estrutura["nome"]} jogou? '))
    lista.clear()                                           # limpar a lista
    for c in range(1, quantidade_partidas + 1):
        lista.append(int(input(f'Quantidades de gols na {c}º partida? ')))
    estrutura['gols'] = lista.copy()
    estrutura['total'] = sum(lista)
    lista_time.append(estrutura.copy())
    pergunta = ' '
    while pergunta not in 'SN':
        pergunta = str(input('Quer continuar [S/N] ').strip().upper()[0])
    if pergunta == 'N':
        break

print('-='*30)                                              # mostrar os resultados
print('cod ', end='')                            # cabeçalho:
for i in estrutura.keys():                         # para cada titulo na estrutura titulos
    print(f'{i:<15}', end='')                   # formatado a esquerda
print()                                             # sempre com quebra linha
print('-'*40)
for k, v in enumerate(lista_time):
    print(f'{k:>3} ', end='')                    # formatação da chave ou titulo alinhado a direita
    for d in v.values():                        # dado e valor
        print(f'{str(d):<15}', end='')
    print()                                         # sempre com quebra linha
print('-'*40)
while True:
    busca = int(input('Mostrar dados de qual jogador? (999 p/ parar) '))              # chave de buscar
    if busca == 999:
        break
    if busca >= len(lista_time):                        # len vai ser a quantidade de dicionaros = jogadores
        print(f'Erro! Não existe jogador com codigo {busca}')
    else:
        print(f' -- Levantamento do jogador {lista_time[busca]["nome"]}:')
        for i, g in enumerate(lista_time[busca]["gols"]):
            print(f'     No jogo {i+1} fez {g} gols')
    print('-='*40)
print('<< Volte sempre >>')

pacote de dawload/projeto  progamas em Python/desafio095 Aprimorando os dicionarios.py


from typing import Dict, Union, List

import tensorflow as tf
from ants_ai.training.neural_network.encoders.encoders import TrainingDataset
from kerastuner import HyperParameters
from ants_ai.training.neural_network.factories.model_factory import ModelFactory, EncodingType
from ants_ai.training.neural_network.factories.model_hyper_parameter import ModelHyperParameter
from ants_ai.training.neural_network.factories.hyper_parameter_factory import HyperParameterFactory

from tensorflow.python.keras import Input
from tensorflow.python.keras.layers import Conv2D, Flatten, Dropout, Dense, MaxPooling2D
from tensorflow.python.keras.models import Model, Sequential

FILTER0_NAME = 'filter_0'
FILTER1_NAME = 'filter_1'
FILTER2_NAME = 'filter_2'
MAX_POOL_SIZE0 = 'maxpool_size1'
MAX_POOL_SIZE1 = 'maxpool_size2'
MAX_POOL_SIZE2 = 'maxpool_size3'
DENSE_NAME = 'dense'
LEARNING_RATE_NAME = 'learning_rate'

# Helpful link
# https://www.quora.com/How-can-I-calculate-the-size-of-output-of-convolutional-layer
class Conv2DMaxPoolModelFactory(ModelFactory):
    def __init__(self, bot_name: str):
        super().__init__('conv_2d_maxpool', bot_name, {
            FILTER0_NAME: ModelHyperParameter(FILTER0_NAME, 32, False),
            FILTER1_NAME: ModelHyperParameter(FILTER1_NAME, 64, False),
            FILTER2_NAME: ModelHyperParameter(FILTER2_NAME, 128, False),
            MAX_POOL_SIZE0: ModelHyperParameter(MAX_POOL_SIZE0, 2, True),
            MAX_POOL_SIZE1: ModelHyperParameter(MAX_POOL_SIZE1, 2, True),
            MAX_POOL_SIZE2: ModelHyperParameter(MAX_POOL_SIZE2, 2, True),
            DENSE_NAME: ModelHyperParameter(DENSE_NAME, 64, False),
            LEARNING_RATE_NAME: ModelHyperParameter(LEARNING_RATE_NAME, 0.001, False)
        })

    def encode_games(self, game_paths: List[str]) -> TrainingDataset:
        return self.encode_game_states(game_paths, EncodingType.ANT_VISION_2D)

    def construct_model(self, tuned_params: Dict[str, Union[int, float]], hps: HyperParameters = None) -> Model:
        hpf = HyperParameterFactory(self.default_parameters_values, tuned_params, hps)
        filter_0 = hpf.get_choice(FILTER0_NAME, [4, 8, 16, 32])
        filter_1 = hpf.get_choice(FILTER1_NAME, [32, 48, 64])
        filter_2 = hpf.get_choice(FILTER2_NAME, [64, 96, 128])
        max_pool_0 = hpf.get_choice(MAX_POOL_SIZE0, [1, 2])
        max_pool_1 = hpf.get_choice(MAX_POOL_SIZE1, [1, 2])
        max_pool_2 = hpf.get_choice(MAX_POOL_SIZE2, [1, 2, 4, 8])
        dense = hpf.get_int(DENSE_NAME, lambda default: hps.Int(DENSE_NAME, 32, 128, step=8, default=default))
        lr = hpf.get_choice(LEARNING_RATE_NAME, [1e-2, 1e-3, 1e-4])

        model = Sequential([
            Input(name='Input', shape=(12, 12, 7)),
            Conv2D(filter_0, 2, strides=1, activation=tf.nn.relu, name='Conv2D_0'),
            MaxPooling2D(max_pool_0, name='MaxPool_0'),

            Conv2D(filter_1, 3, strides=1, activation=tf.nn.relu, name='Conv2D_1'),
            MaxPooling2D(max_pool_1, name='MaxPool_1'),
            # Conv2D(self.get_param_value(FILTER2_NAME, tuned_params), 2, strides=1, activation=tf.nn.relu,
            #       name='Conv2D_2'),
            # MaxPooling2D(self.get_param_value(MAX_POOL_SIZE2, tuned_params), name='MaxPool_2'),
            Flatten(name='Flatten'),
            Dropout(0.1, name='Dropout'),
            Dense(dense, activation=tf.nn.relu, name='dense'),
            Dense(5, activation=tf.nn.softmax, name='Output'),
        ])
        loss_fn = tf.keras.losses.CategoricalCrossentropy()
        opt = tf.keras.optimizers.Adam(learning_rate=lr)
        model.compile(optimizer=opt,
                      loss=loss_fn,
                      metrics=[tf.keras.metrics.categorical_accuracy])
        return model


from django.shortcuts import render, get_object_or_404, redirect

from .models import Projecten
from .models import Vloeren
from .models import Bedrijven
from .models import Datum_Document
from .models import Controlemomenten_gepland

from django.contrib.auth.decorators import login_required
from django.views.decorators.http import require_http_methods
from django.views.decorators.csrf import csrf_protect

from .forms import ProjectenForm
from .forms import BedrijvenForm
from .forms import VloerenForm
from .forms import Datum_DocumentForm
from .forms import Controlemomenten_geplandForm
# from .forms import Update_documentForm

from django.db.models import Max
from django.db.models import F

#from django.forms import formset_factory

alle_bedrijven = Bedrijven.objects.all()

# Create your views here.
def projecten(request):
    alle_projecten = Projecten.objects.all()
    return render(request, 'database/projecten.html', {'alle_projecten': alle_projecten})

def vloeren(request):
    alle_vloeren = Vloeren.objects.all()
    alle_vloeren_laatste = Vloeren.objects.annotate(laatste=Max('datum_document__datum')).filter(datum_document__datum=F('laatste'))

    return render(request, 'database/vloeren.html', {'alle_vloeren_laatste': alle_vloeren_laatste})

def voortgang(request):
    alle_vloeren = Vloeren.objects.all()
    return render(request, 'database/voortgang.html', {'alle_vloeren': alle_vloeren})

def statusbeheer(request):
    alle_vloeren = Vloeren.objects.all()
    return render(request, 'database/statusbeheer.html', {'alle_vloeren': alle_vloeren})

def bedrijven(request):
    alle_bedrijven = Bedrijven.objects.all()
    return render(request, 'database/bedrijven.html', {'alle_bedrijven': alle_bedrijven})

def projectinvoeren(request):
    if request.method == 'POST':
        formprojecten = ProjectenForm(request.POST)
        if formprojecten.is_valid():
            formprojecten.save()
            return redirect('projecten')
    else:
        formprojecten = ProjectenForm()
    return render(request, 'database/projectinvoeren.html', {'formprojecten': formprojecten})

#@login_required
def documentinvoeren(request):

    if request.method == 'POST':
        formvloeren = VloerenForm(request.POST, request.FILES)
        nieuwe_datum = request.POST["nieuwe_datum"]
        # formdatumdocument = Datum_DocumentForm(request.POST, request.FILES)
        # if formvloeren.is_valid() and formdatumdocument.is_valid():
        if formvloeren.is_valid():
            new_vloer = formvloeren.save()
            # new_datum_doc = formdatumdocument.save(commit=False)
            datum = Datum_Document.objects.create(datum=nieuwe_datum, vloeren=new_vloer)
            # datum.vloeren = new_vloer
            # new_datum_doc.save()
            return redirect('documentinvoeren')
    else:
        formdatumdocument = Datum_DocumentForm()
        formvloeren = VloerenForm()

    return render(request, 'database/documentinvoeren.html', {'formvloeren': formvloeren, 'formdatumdocument': formdatumdocument})

def bedrijfinvoeren(request):

    if request.method == 'POST':
        formbedrijven = BedrijvenForm(request.POST)
        if formbedrijven.is_valid():
            formbedrijven.save()
            return redirect('bedrijven')
    else:
        formbedrijven = BedrijvenForm()
    return render(request, 'database/bedrijfinvoeren.html', {'formbedrijven': formbedrijven, 'alle_bedrijven': alle_bedrijven})

#@login_required
# https://docs.djangoproject.com/en/3.1/topics/http/decorators/#django.views.decorators.http.require_http_methods
@require_http_methods(['POST'])
@csrf_protect
def delete_bedrijf(request):
    # https://docs.djangoproject.com/en/3.1/ref/request-response/#django.http.QueryDict.getlist
    object_ids = request.POST.getlist('id')
    object_list = Bedrijven.objects.filter(id__in=object_ids)
    # print(object_list)
    object_list.delete()
    return redirect('bedrijven')

#@login_required
@require_http_methods(['POST'])
@csrf_protect
def delete_document(request):
    object_ids = request.POST.getlist('id')
    object_list = Vloeren.objects.filter(id__in=object_ids)
    # print(object_list)
    object_list.delete()
    return redirect('vloeren')

# @require_http_methods(['POST'])
@csrf_protect
#@login_required
def update_document(request):
    alle_vloeren = Vloeren.objects.all()
    alle_vloeren_laatste = Vloeren.objects.annotate(laatste=Max('datum_document__datum')).filter(datum_document__datum=F('laatste'))

    if request.method == 'POST':
        datumnew = request.POST.get('nieuwe datum')
        object_ids = request.POST.getlist('id')  #get id's from checked checkboxes template
        object_list = Vloeren.objects.filter(id__in=object_ids)
        #print(object_list)
        #formdatumdocument = Datum_DocumentForm(request.POST)
        #if formdatumdocument.is_valid():
        #    new_datum_doc = formdatumdocument.save(commit=False)
            #new_datum_doc.vloeren = GET VLOEROBJECT FROM OBJECT_LIST (new_vloer = formvloeren.save())
        for object in object_list:
            # new_datum_doc.vloer = object
            # new_datum_doc.save()
            object.datum_document_set.create(datum = datumnew)
    # else:
        # formdatumdocument = Datum_DocumentForm
    return render(request, 'database/document_updaten.html', {'alle_vloeren': alle_vloeren, 'alle_vloeren_laatste': alle_vloeren_laatste})

# def update_document2(request):
#     # alle_vloeren = Vloeren.objects.all()
#     alle_vloeren_laatste = Vloeren.objects.annotate(laatste=Max('datum_document__datum')).filter(datum_document__datum=F('laatste'))
#     updateform = Update_documentForm()

#     return render(request, "database/document_updaten2.html", {'alle_vloeren_laatste': alle_vloeren_laatste, 'updateform': updateform})


# Definition for singly-linked list.
# class ListNode:
#     def __init__(self, val=0, next=None):
#         self.val = val
#         self.next = next
class Solution:
    def pairSum(self, head: Optional[ListNode]) -> int:
        nodes, l, maxx = [], 0, 0
        while head:
            nodes.append(head.val)
            head = head.next

        while l < len(nodes):
            maxx = max(maxx, nodes[l]+nodes[-1])
            l+=1
            nodes.pop()

        return maxx


# input file for VerticalExcitation.py
import numpy as np
import numpy.typing as npt

def construct_conn_array(n: int):
    """
    connectivity array constructor for benchmark case where tether is fixed at both ends and is deflected by
    perpendicular flow. Tether experiences no gravity.
    :param n: number of particles N
    :return: connectivity array
    """
    con_array = np.zeros((n, n))
    a1 = list(range(0, n-1))
    a2 = list(range(1, n))
    index = list(zip(a1, a2))
    for i in index:
        con_array[i] = 1
        con_array[i[::-1]] = 1

    return con_array

def construct_initial_conditions_array(n: int, l: float, fixed: npt.ArrayLike, m_tether_segment: float):
    """
    initial position & velocity array constructor
    :param n: number of particles N
    :return: array of initial particle states [ p1:[position, velocity, mass, particle fixed], ...]
    """
    dl = l/(n-1)

    x = [[0, 0, l-i*dl] for i in range(n)]
    v = [[0, 0, 0] for j in range(n+1)]
    m = [[m_tether_segment] for l in range(n)]  # can be calc'd more generally by using the connectivity array rows sum
    m[0][0] /= 2
    m[-1][0] /= 2
    return list(zip(x, v, m, fixed))

# physical constants
g = 0            # [m/s^2]    gravitational acceleration
K = 6E4             # [N/m]      stiffness of the rope
AE = 6E12           # [N]        Area*Young's modulus
c = 10             # [N/m*s^-1] damping coefficient of the system  #1E2
L = 10              # [m]        length of the rope
rho_rope = 0.1      # [kg/m]     LINEAR density of the rope
diam_bridle = 0.02  # [m]        diameter of bridle lines
c_d_bridle = 1.05   # [-]        drag-coefficient of bridles
rho_air = 1.225
v_wind = 5           # [m/s] uniform wind field, aligned with x-axis

# simulation parameters
N = 10                # [-]         #particles, [0=block,1:N-1 = rope, N = ceiling particle]
fixed = np.zeros(N)
fixed[0] = fixed[-1] = 1
n_steps = 2000

# solver settings
dt = 0.005               # timestep
stoptime = 6            # limit of time
abserr = 1e-4           # absolute tolerance
relerr = 1e-3           # relative tolerance

# CALCULATED input parameters
l0 = L/(N-1)                # [m]  distance between the particles in the rope (rest length)
# k_segment = AE/l0           # [N/m] stiffness tether segment
k_segment = K*(N-1)
m_segment = l0*rho_rope     # [kg] mass tether segment

if c == 0:                  # If no value for damping coefficient is set, calculate c for critically damped system
    # k_tether = AE/L
    k_tether = K
    m_total = L*rho_rope
    beta = np.sqrt(k_tether/m_total)
    c_tether = 2*m_total*beta
    c_segment = c_tether*(N-1)
else:
    c_segment = c*(N-1)
c_segm_unequal = 2*np.sqrt(k_segment*m_segment)
c_end_unequal = 2*np.sqrt(k_segment * m_segment)

conn_array = construct_conn_array(N)
initial_cond = construct_initial_conditions_array(N, L, fixed, m_segment)

# tolerance settings from KiteSim
atol = 1e-50
rtol = 1e-5
maxiter = 100000

if __name__ == "__main__":
    # import pandas as pd
    # t = [n * dt for n in range(3)]
    # r1 = np.zeros((30, ))
    # print(r1)
    # v = pd.DataFrame(data=[], index=t, columns=[item for sublist in [[f'vx_{i}', f'vy_{i}', f'vz_{i}'] for i in range(1, N + 1)] for item in sublist], dtype=float)
    # print(v)
    # v.loc[0.00] = r1
    # print(v.loc[0])

    # a = np.array([[0, 0, 0], [0, 1, 0], [1, 0, 1]])
    # b = np.nonzero(np.triu(a))
    # b = np.column_stack((b[0], b[1]))
    # print(b)
    # for indices in b:
    #     print(a[indices[0], indices[1]])

    # print(np.zeros((2, 10, 3, )))
    pass


import argparse
import signal
import threading
import time
from queue import Queue
from typing import List

from watchdog.observers.inotify import InotifyObserver

from tagger import DATA_BASE_PATH
from tagger.dataset.dataset import DatasetHandler, DatasetWorker
from tagger.db.mongo_client import init_db
from tagger.model.model_register import ModelRegister
from tagger.model.model_worker import ModelWorker
from tagger.utils import logger

def stop(threads):
    logger.info("Stopping threads")
    thread_event.set()
    for worker in threads:
        if isinstance(worker, InotifyObserver):
            worker.stop()
        worker.join()
    logger.info("Threads stopped")

def signal_handler(signum, frame):
    logger.info("Received signal: %s", signum)
    main_event.set()

def main(watch: bool = False, ):

    signal.signal(signal.SIGINT, signal_handler)
    signal.signal(signal.SIGTERM, signal_handler)

    model_register = ModelRegister()
    model_register.find_all_models()

    init_db()

    threads = []
    input_queues: List[Queue] = []

    for model_handler in model_register.list_models():
        input_queue = Queue()
        input_queues.append(input_queue)

        worker = ModelWorker(model_handler, input_queue, thread_event)
        worker.start()
        threads.append(worker)

    dataset_worker = DatasetWorker(input_queues, thread_event)
    dataset_worker.start()
    threads.append(dataset_worker)

    if watch:
        observer = InotifyObserver()
        handler = DatasetHandler(input_queues)
        observer.schedule(handler, DATA_BASE_PATH, recursive=True)
        observer.start()
        threads.append(observer)
        logger.info("Started file system watchdog")
    else:
        dataset_worker.join()
        main_event.set()
        logger.info("Dataset processing completed")

    try:
        while not main_event.is_set():
            time.sleep(.1)
        stop(threads)
    except KeyboardInterrupt:
        # Ctrl-C handling and send kill to threads
        main_event.set()
        stop(threads)

parser = argparse.ArgumentParser(description='py-photo-tagger cli arguments')
parser.add_argument('-w', '--watch', dest='watch', action='store_true', help='Watch filesystem for changes')

main_event = threading.Event()
thread_event = threading.Event()

if __name__ == '__main__':
    args = parser.parse_args()

    main(args.watch)


class Node:
    def __init__(self, data, next=None):
        self.data = data
        self.next = next

class NodeMgmt:
    def __init__(self, data):
        self.head = Node(data)

    def add(self, data): # 맨끝에 추가
        if self.head == '': # 헤드 없다면 노드 새로 생성해서 헤드로
            self.head = Node(data)
        else:
            node = self.head
            while node.next:
                node = node.next
            node.next = Node(data) # 맨마지막 노드의 주소에 새로운 노드 연결 및 생성

    def desc(self):
        node = self.head
        while node:
            print(node.data)
            node = node.next

    def delete(self, data):
        if self.head == '':
            print("해당 값을 가진 노드가 없습니다.")
            return

        if self.head.data == data:
            temp = self.head
            self.head = self.head.next
            del temp
        else:
            node = self.head
            while node.next:
                if node.next.data == data:
                    temp = node.next
                    node.next = node.next.next # 마지막 노드도 가능
                    del temp
                    return
                else:
                    node = node.next

linkedlist1 = NodeMgmt(0)
linkedlist1.desc()

for data in range(1,11):
    linkedlist1.add(data)
linkedlist1.desc()

print("3 삭제")

linkedlist1.delete(3)
linkedlist1.desc()

print("0 삭제")

linkedlist1.delete(0)
linkedlist1.desc()

print("10 삭제")

linkedlist1.delete(10)
linkedlist1.desc()


import re
from collections import Counter
from pathlib import Path
from shutil import copyfile
from typing import Set, List, Literal, Iterable, Optional

import pandas
from sortedcontainers import SortedDict
from tqdm import tqdm

from .base import BaseTask, CorporaTaskMixin
from .filters.base import CandidatesPairCSV
from .imports import FileID
from .tokenize import TokenizedWordsCSV
from ..utils import logger
from ..workspace import Workspace, WorkspaceCSV

def load_group_words(group: Set[FileID], workspace: Workspace) -> Counter:
    tokenized_folder = workspace.root_path / Path("datasets/tokenized/per_text/")

    group_words = Counter()
    for file_id in group:
        tokenized_file = TokenizedWordsCSV(tokenized_folder / Path(f"{file_id}.csv"))
        try:
            group_words.update(tokenized_file.to_dict())
        except FileNotFoundError:
            logger.warning(f"Couldn't find tokenized text file {file_id}.csv")
            continue

    return group_words

class CorporaCreationTask(BaseTask):
    """Creates corpora based on the list of tokenized words for each text
    in the dataset and the text files families. The output words list are
    compted from the intersection of the words of the texts from the families'
    groups"""
    requires = [
        "datasets/tokenized/all.csv",
        "datasets/families/*/*.txt"
    ]

    creates = [
        "corpora/",
        "corpora/tokenized/*.csv",
    ]

    @staticmethod
    def sort_families_folders(folders: Iterable[Path]) -> List[Path]:
        return sorted(folders, key=lambda x: int(x.stem.split("_")[1]),
                      reverse=True)

    def run(self, workspace: Workspace):
        workspace.corpora.mkdir(parents=True, exist_ok=True)

        logger.info("Building families words lists...")
        families_folder = workspace.datasets / Path("families/")
        # Families *have* to be sorted
        pbar = tqdm(self.sort_families_folders(families_folder.iterdir()))
        previous_group_words: Set[str] = set()
        for family_folder in pbar:
            family_folder: Path
            assert family_folder.is_dir()
            family_id = int(re.fullmatch(r"family_([0-9]+)", family_folder.name)[1])
            assert len(list(family_folder.iterdir())) == family_id

            family_words_dict = None
            groups_paths = self.sort_families_folders(list(family_folder.iterdir()))
            for group_filepath in groups_paths:
                pbar.set_description(f"Family {family_id}: {group_filepath.name}")

                assert group_filepath.is_file()
                with open(group_filepath) as group_file:
                    group = set(file_id for file_id in group_file.read().split("\n") if file_id)
                group_words_dict = load_group_words(group, workspace)

                # the first group is taken as base for the intersection
                if family_words_dict is None:
                    family_words_dict = group_words_dict
                else:
                    # else we only keep the words set that intersect
                    # and increment their counts with the currentgroup's count
                    words_intersection = (set(group_words_dict) & set(family_words_dict))
                    for word in list(family_words_dict):
                        if word in words_intersection:
                            family_words_dict[word] += group_words_dict[word]
                        else:
                            del family_words_dict[word]

            # removing words from "previous" family
            len_before = len(family_words_dict)
            for word in previous_group_words:
                if word in family_words_dict:
                    del family_words_dict[word]
            previous_group_words.update(set(family_words_dict.keys()))

            logger.debug(f"Removed {len_before - len(family_words_dict)} words "
                         f"from smaller corpora in corpus {family_id}")

            logger.debug(f"Writing words list for family {family_id}")
            tokenized_corpora_folder = workspace.corpora / Path("tokenized/")
            tokenized_corpora_folder.mkdir(parents=True, exist_ok=True)
            family_words_csv = TokenizedWordsCSV(tokenized_corpora_folder /
                                                 Path(f"corpus_{family_id}.csv"))
            # sorting dict by word count (decreasing)
            family_words_dict = dict(sorted(family_words_dict.items(),
                                            key=lambda item: item[1],
                                            reverse=True))
            with family_words_csv.dict_writer as dict_writer:
                dict_writer.writeheader()
                for word, count in family_words_dict.items():
                    dict_writer.writerow({
                        "word": word, "count": count
                    })

class ZeroSpeechCSV(WorkspaceCSV):
    header = ["id", "filename", "voice", "frequency", "word", "phones", "length", "correct"]

    def __init__(self, file_path: Path):
        super().__init__(file_path, separator=",", header=self.header)

class BuildZeroSpeechTestSetsTask(BaseTask, CorporaTaskMixin):
    requires = [
        "corpora/wuggy_pairs/*",
        "synth/audio/phonetic/",
    ]

    def __init__(self,
                 output_folder : Path,
                 real_word_synth: Literal["text", "phonetic"],
                 for_corpus: Optional[int] = None):
        super().__init__()
        self.real_word_synth = real_word_synth
        self.for_corpus = for_corpus
        self.output_folder  = output_folder

    def build_frequencies_csv(self,
                              zr_corpus_folder: Path,
                              corpus_id: int,
                              workspace: Workspace):
        family_folder = workspace.datasets / Path(f"families/family_{corpus_id}")
        corpus_csv = TokenizedWordsCSV(workspace.corpora / Path(f"tokenized/corpus_{corpus_id}.csv"))
        corpus_words = set(corpus_csv.to_dict().keys())

        group_paths = sorted(family_folder.iterdir(), key=lambda p: int(p.stem.split("_")[1]))
        freqs_df = pandas.DataFrame()
        for group_filepath in tqdm(group_paths):

            with open(group_filepath) as group_file:
                file_ids = {f_id for f_id in group_file.read().split("\n") if f_id}
            group_words_freqs = load_group_words(file_ids, workspace)

            group_words_freqs = SortedDict({word: freq
                                            for word, freq in group_words_freqs.items()
                                            if word in corpus_words})
            if freqs_df.index.empty:
                freqs_df.index = list(group_words_freqs.keys())

            freqs_df[group_filepath.name] = list(group_words_freqs.values())

        freqs_df.to_csv(zr_corpus_folder / Path("frequencies.csv"),
                        sep=",")

    def build_zr_testset(self,
                         wuggy_pairs_path: Path,
                         zr_corpus_folder: Path,
                         workspace: Workspace):
        zr_audio_folder = zr_corpus_folder / Path("ogg")
        zr_audio_folder.mkdir(parents=True, exist_ok=True)
        zr_csv_path = zr_corpus_folder / Path("gold.csv")
        words_dataset_freqs = TokenizedWordsCSV(workspace.tokenized / "all.csv").to_dict()

        phonetic_synth_folder = workspace.synth / Path("audio/phonetic/")
        text_synth_folder = workspace.synth / Path("audio/text/")
        voices = [p.name for p in phonetic_synth_folder.iterdir()]

        corpus_wuggy_pairs_csv = CandidatesPairCSV(wuggy_pairs_path)
        zr_corpus_csv = ZeroSpeechCSV(zr_csv_path)

        pbar = tqdm(total=corpus_wuggy_pairs_csv.lines_count * len(voices))

        with zr_corpus_csv.dict_writer as dict_writer:
            dict_writer.writeheader()
            for pair_id, (word, phonetic, fake_phonetic) in enumerate(corpus_wuggy_pairs_csv):
                if self.real_word_synth == "text":
                    word_filename = word
                else:
                    word_filename = phonetic.replace(" ", "_")
                fake_word_filename = fake_phonetic.replace(" ", "_")

                for voice in voices:
                    # writing pair entries in the "gold" csv file
                    dict_writer.writerow({
                        "id": pair_id + 1,
                        "filename": f"{word_filename}-{voice}",
                        "voice": voice,
                        "frequency": words_dataset_freqs[word],
                        "word": word,
                        "phones": phonetic,
                        "length": len(phonetic.split(" ")),
                        "correct": 1
                    })
                    dict_writer.writerow({
                        "id": pair_id + 1,
                        "filename": f"{fake_word_filename}-{voice}",
                        "voice": voice,
                        "frequency": "",
                        "word": None,
                        "phones": fake_phonetic,
                        "length": len(fake_phonetic.split(" ")),
                        "correct": 0
                    })

                    # copying audio files for the pair
                    if self.real_word_synth == "text":
                        word_audio_path = text_synth_folder / Path(f"{voice}/{word_filename}.ogg")
                    else:
                        word_audio_path = phonetic_synth_folder / Path(f"{voice}/{word_filename}.ogg")
                    word_cp_path = zr_audio_folder / Path(f"{word_filename}-{voice}.ogg")
                    fake_word_audio_path = phonetic_synth_folder / Path(f"{voice}/{fake_word_filename}.ogg")
                    fake_word_cp_path = zr_audio_folder / Path(f"{fake_word_filename}-{voice}.ogg")

                    if not word_cp_path.is_file():
                        copyfile(word_audio_path, word_cp_path)
                    if not fake_word_cp_path.is_file():
                        copyfile(fake_word_audio_path, fake_word_cp_path)

                    pbar.update()

    def run(self, workspace: Workspace):
        zr_folder = self.output_folder
        zr_folder.mkdir(parents=True, exist_ok=True)
        wuggy_pairs_folder = workspace.corpora / Path("wuggy_pairs")

        corpora = self.find_corpora(wuggy_pairs_folder)
        if self.for_corpus is not None:
            corpora = [(corpus_id, corpus_path)
                       for corpus_id, corpus_path in corpora
                       if corpus_id == self.for_corpus]

        for corpus_id, corpus_csv_path in corpora:
            corpus_zr_folder = zr_folder / Path(f"testset_{corpus_id}")
            logger.info(f"For corpus {corpus_id}")
            logger.info("Building corpus table and aggregating audio files")
            self.build_zr_testset(corpus_csv_path, corpus_zr_folder, workspace)
            logger.info("Building corpus word frequencies table")
            self.build_frequencies_csv(corpus_zr_folder, corpus_id, workspace)


#!/usr/bin/env python
# _*_ coding:utf-8 _*_
# ============================================
# @Time     : 2020/05/12 15:40
# @Author   : WanDaoYi
# @FileName : backbone.py
# ============================================

import keras.layers as kl
from m_rcnn import common
from config import cfg

def resnet_graph(input_image, architecture, stage5=False):
    """
        resNet 背骨图，没什么好说的，数好参数就好了。
    :param input_image: input image info
    :param architecture: Can be resNet50 or resNet101
    :param stage5: Boolean. If False, stage5 of the network is not created
    :return: [c1, c2, c3, c4, c5]
    """
    train_flag = cfg.COMMON.TRAIN_FLAG
    assert architecture in ["resNet50", "resNet101"]
    # Stage 1
    x = kl.ZeroPadding2D((3, 3))(input_image)
    x = kl.Conv2D(64, (7, 7), strides=(2, 2), name='conv1', use_bias=True)(x)
    x = common.BatchNorm(name='bn_conv1')(x, training=train_flag)
    x = kl.Activation('relu')(x)
    c1 = x = kl.MaxPooling2D((3, 3), strides=(2, 2), padding="same")(x)

    # Stage 2
    x = common.conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1), train_flag=train_flag)
    x = common.identity_block(x, 3, [64, 64, 256], stage=2, block='b', train_flag=train_flag)
    c2 = x = common.identity_block(x, 3, [64, 64, 256], stage=2, block='c', train_flag=train_flag)

    # Stage 3
    x = common.conv_block(x, 3, [128, 128, 512], stage=3, block='a', train_flag=train_flag)
    x = common.identity_block(x, 3, [128, 128, 512], stage=3, block='b', train_flag=train_flag)
    x = common.identity_block(x, 3, [128, 128, 512], stage=3, block='c', train_flag=train_flag)
    c3 = x = common.identity_block(x, 3, [128, 128, 512], stage=3, block='d', train_flag=train_flag)

    # Stage 4
    x = common.conv_block(x, 3, [256, 256, 1024], stage=4, block='a', train_flag=train_flag)
    block_count = {"resNet50": 5, "resNet101": 22}[architecture]
    for i in range(block_count):
        x = common.identity_block(x, 3, [256, 256, 1024], stage=4, block=chr(98 + i), train_flag=train_flag)
    c4 = x

    # Stage 5
    if stage5:
        x = common.conv_block(x, 3, [512, 512, 2048], stage=5, block='a', train_flag=train_flag)
        x = common.identity_block(x, 3, [512, 512, 2048], stage=5, block='b', train_flag=train_flag)
        c5 = common.identity_block(x, 3, [512, 512, 2048], stage=5, block='c', train_flag=train_flag)
    else:
        c5 = None

    return [c1, c2, c3, c4, c5]


#!/usr/bin/python3
# -*- coding: utf-8 -*-

"""
Управление классами ядра системы
"""

from os import path
import json
import logging

from config import SETTINGS
from config import TEST_PATH_RESULTS, WRONG_AUTO_MINFIN_TESTS_FILE
from constants import ERROR_NO_DOCS_FOUND, ERROR_REQUEST_CONTAINS_BAD_WORD
from core.answer_object import CoreAnswer
from core.cube_classifier import CubeClassifier
from core.cube_docs_processing import CubeAnswer
from core.cube_docs_processing import CubeProcessor
from core.cube_or_minfin_classifier import CubeOrMinfinClassifier
from core.minfin_docs_processing import MinfinProcessor
from core.solr import Solr
from core.support_library import group_documents
from core.support_library import process_cube_answer
from core.support_library import process_server_response
from core.support_library import send_request_to_server
from model_manager import MODEL_CONFIG
from text_preprocessing import TextPreprocessing
import logs_helper  # pylint: disable=unused-import

class DataRetrieving:
    """
    Модуль связывающие в себе результаты работы Apache Solr,
    и классов обработки его выдачи - CubeProcessor и MinfinProcessor
    """

    TPP = TextPreprocessing(label='DATRET', delete_question_words=False)

    # хранение данных в памяти
    _minfin_auto_wrong_data = None

    @staticmethod
    def get_data(user_request: str, request_id: str):
        """API метод к ядру системы"""

        core_answer = CoreAnswer()
        core_answer.user_request = user_request

        if user_request.lower().startswith("кто будет"):
            return core_answer

        norm_user_request = DataRetrieving._preprocess_user_request(
            core_answer.user_request,
            request_id
        )

        correct_answer_num = DataRetrieving._process_exact_minfin_answers(
            user_request
        )

        # обработка плохих слов
        badcount = norm_user_request.count('<censored>')
        if badcount > 0:
            core_answer.bad_content = True
            core_answer.message = ERROR_REQUEST_CONTAINS_BAD_WORD

            logging.info(
                'Query_ID: {}\tMessage: Запрос содержал {}'.format(
                    request_id,
                    'плохое слово' if badcount == 1 else 'плохие слова'
                )
            )

            return core_answer

        # получение результатов поиска от Apache Solr в JSON-строке
        solr_response = Solr.get_data(
            norm_user_request,
            request_id,
            SETTINGS.SOLR_MAIN_CORE
        )['response']

        # Если хотя бы 1 документ найден:
        if solr_response['numFound']:
            minfin_docs, cube_data = group_documents(
                solr_response['docs'],
                core_answer.user_request,
                norm_user_request,
                request_id
            )

            minfin_answers = MinfinProcessor.get_data(minfin_docs)

            clf = CubeClassifier.inst()
            best_prediction = tuple(
                clf.predict_proba(core_answer.user_request)
            )[0]

            ans_type = None

            # ручная проверка, актуальная для вопросов по pretty_feedback
            clf_status, cube = DataRetrieving._manual_cube_classification(
                norm_user_request
            )

            if clf_status:
                best_prediction = (cube, 0.99)
                ans_type = 'cube'

            cube_answers, cube_confidence = CubeProcessor.get_data(
                cube_data, best_prediction)

            logging.info(
                "Query_ID: {}\tMessage: Найдено {} докумета(ов) по кубам и {} по Минфину".format(
                    request_id,
                    len(cube_answers),
                    len(minfin_answers)
                )
            )

            answers = DataRetrieving._sort_answers(
                minfin_answers,
                cube_answers,
                correct_answer_num,
                ans_type
            )

            DataRetrieving._format_core_answer(
                answers,
                request_id,
                core_answer,
                cube_confidence
            )
        else:
            # Обработка случая, когда документы не найдены
            core_answer.message = ERROR_NO_DOCS_FOUND
            logging.info(
                'Query_ID: {}\tMessage: Документа не найдены'.format(
                    request_id)
            )

        return core_answer

    @staticmethod
    def _process_exact_minfin_answers(user_request: str):
        if MODEL_CONFIG["use_local_file_processing_for_minfin"]:
            minfin_auto_wrong_tests = DataRetrieving._minfin_auto_wrong_questions()
            user_request = user_request.lower()

            user_request = user_request.replace('?', '')

            # сработает для формулировки с любым порядком слов
            for key in minfin_auto_wrong_tests.keys():
                if set(user_request.split()) == set(key.split()):
                    return minfin_auto_wrong_tests.get(key, 0)

    @staticmethod
    def _minfin_auto_wrong_questions():
        if not DataRetrieving._minfin_auto_wrong_data:
            minfin_wrong_auto_tests_file = path.join(
                TEST_PATH_RESULTS, WRONG_AUTO_MINFIN_TESTS_FILE
            )

            with open(minfin_wrong_auto_tests_file, 'r', encoding='utf-8') as file:
                DataRetrieving._minfin_auto_wrong_data = json.loads(
                    file.read())

        return DataRetrieving._minfin_auto_wrong_data

    @staticmethod
    def _preprocess_user_request(user_request: str, request_id: str):
        """Предобработка запроса пользователя"""

        # нормализация запроса пользователя
        norm_user_request = DataRetrieving.TPP(user_request, request_id)

        if MODEL_CONFIG["delete_repeating_words_in_request"]:
            norm_user_request = DataRetrieving._set_user_request(
                norm_user_request, request_id
            )

        if MODEL_CONFIG["enable_repetition_for_short_request"]:
            norm_user_request = DataRetrieving._multiple_user_request(
                norm_user_request, request_id
            )

        return norm_user_request

    @staticmethod
    def _set_user_request(norm_user_request: str, request_id: str):
        """
        Удаление повторяющихся слов и увеличение длины запроса
        при необходимости
        """

        request_words = norm_user_request.split()
        unique_of_request_words = set(request_words)

        # удаление повторяющихся слов, чтобы пользователи не читерили
        if len(unique_of_request_words) != len(request_words):
            norm_user_request = ' '.join(unique_of_request_words)

            logging.info(
                "Query_ID: {}\tMessage: Удалено {} повторяющихся слов".format(
                    request_id,
                    len(request_words) - len(unique_of_request_words)
                )
            )

        return norm_user_request

    @staticmethod
    def _multiple_user_request(norm_user_request: str, request_id: str):
        """
        Дублирование коротких запросов
        """

        short_question_threshold = MODEL_CONFIG["short_request_threshold"]
        multiplier = MODEL_CONFIG["repetition_num_for_short_request"]
        request_len = len(norm_user_request.split())

        if request_len <= short_question_threshold:
            norm_user_request = ' '.join(
                [norm_user_request.strip()] * multiplier
            )

            logging.info(
                "Query_ID: {}\tMessage: Запрос из {} слов был "
                "удлинен в {} раза".format(
                    request_id,
                    request_len,
                    multiplier
                )
            )

        return norm_user_request

    @staticmethod
    def _sort_answers(
            minfin_answers: list,
            cube_answers: list,
            correct_answer_num: str,
            ans_type: str
    ):
        """Совокупное ранжирование ответов по кубам и минфину"""

        # Если по минфину найден только 1 ответ
        if not isinstance(minfin_answers, list):
            minfin_answers = [minfin_answers]

        # Если по куба найден только 1 ответ
        if not isinstance(cube_answers, list):
            cube_answers = [cube_answers]

        # Фильтрация выборки в порядке убывания по score
        all_answers = sorted(
            cube_answers + minfin_answers,
            key=lambda ans: ans.get_score(),
            reverse=True
        )

        if all_answers:
            DataRetrieving._first_place_right_type(
                all_answers,
                ans_type
            )

            if MODEL_CONFIG["use_local_file_processing_for_minfin"]:
                DataRetrieving._first_place_exact_minfin_answer(
                    all_answers,
                    correct_answer_num
                )

        return all_answers

    @staticmethod
    def _first_place_right_type(all_answers: list, ans_type: str):

        if not ans_type:
            user_request = all_answers[0].user_request

            clf = CubeOrMinfinClassifier.inst()
            prediction = tuple(clf.predict_proba(user_request))[0]
            ans_type = prediction[0].lower()

        if all_answers[0].type != ans_type:
            for elem in list(all_answers):
                if elem.type == ans_type:
                    all_answers.remove(elem)
                    all_answers.insert(0, elem)

                    logging.info(
                        "Query_ID: {}\tMessage: Главный ответ был "
                        "сменен на тип: {}".format(
                            elem.request_id,
                            ans_type
                        )
                    )

                    break
        else:
            logging.info(
                "Query_ID: {}\tMessage: Тип главного ответа совпадает с "
                "типом из классификатора".format(all_answers[0].request_id)
            )

    @staticmethod
    def _first_place_exact_minfin_answer(all_answers: list, correct_number: str):
        if correct_number:
            for answer in list(all_answers):
                if (answer.type == 'minfin' and
                        answer.number == correct_number):
                    all_answers.remove(answer)
                    all_answers.insert(0, answer)

                    if answer.get_score() < MODEL_CONFIG["relevant_minfin_main_answer_threshold"]:
                        score = MODEL_CONFIG["relevant_minfin_main_answer_threshold"]

                        logging.info(
                            "Query_ID: {}\tMessage: Score был увеличен с {} до {}".format(
                                all_answers[0].request_id,
                                answer.get_score(),
                                score
                            )
                        )

                        answer.score = score

                    logging.info(
                        "Query_ID: {}\tMessage: Главный ответ был сменен на основе"
                        " логов некорректных автоматических тестов".format(
                            all_answers[0].request_id
                        )
                    )

                    break

    @staticmethod
    def _format_core_answer(
            answers: list,
            request_id: str,
            core_answer: CoreAnswer,
            cube_confidence: str
    ):
        """Формирование структуры финального ответа"""

        # Предельное количество "смотри также"
        THRESHOLD = 5

        if answers:
            DataRetrieving._process_main_answer(
                core_answer,
                answers,
                request_id,
                cube_confidence
            )

            starting_from = 1
            if not core_answer.answer:
                starting_from = 0
                THRESHOLD -= 1

            DataRetrieving._process_more_answers(
                core_answer,
                answers[starting_from:THRESHOLD + 1],
                request_id
            )

        return core_answer

    @staticmethod
    def _process_main_answer(
            core_answer: CoreAnswer,
            answers: list,
            request_id: str,
            cube_confidence: str
    ):
        """Форматирование главного ответа"""

        # Выбор главного ответа
        core_answer.answer = answers[0]

        # Если главный ответ по кубам
        if isinstance(core_answer.answer, CubeAnswer):
            core_answer.status = True
            core_answer.confidence = cube_confidence

            logging.info(
                "Query_ID: {}\tMessage: Главный ответ - "
                "ответ по кубу - {} - {}".format(
                    request_id,
                    core_answer.answer.get_score(),
                    core_answer.answer.mdx_query
                ))

            response = send_request_to_server(
                core_answer.answer.mdx_query,
                core_answer.answer.cube
            )

            # ответ с сервера
            value = process_server_response(core_answer.answer, response)

            # форматирование ответа при его наличии
            if value is not None:
                process_cube_answer(core_answer.answer, value)
        # Если главный ответ по минфину
        else:
            # фильтр по релевантности на минфин
            if (core_answer.answer.get_score() >=
                    MODEL_CONFIG["relevant_minfin_main_answer_threshold"]):

                core_answer.status = True

                logging.info(
                    "Query_ID: {}\tMessage: Главный точный ответ - "
                    "ответ по Минфину - {} - {}".format(
                        request_id,
                        core_answer.answer.get_score(),
                        core_answer.answer.number
                    ))
            elif (core_answer.answer.get_score() >=
                  MODEL_CONFIG["minfin_main_answer_confidence_threshold"]):

                core_answer.status = True
                core_answer.confidence = False

                logging.info(
                    "Query_ID: {}\tMessage: Главный возможный ответ - "
                    "ответ по Минфину - {} - {}".format(
                        request_id,
                        core_answer.answer.get_score(),
                        core_answer.answer.number
                    ))
            else:
                # Обнуление найденого ответа
                core_answer.answer = None

                logging.info(
                    "Query_ID: {}\tMessage: Главный ответ по Минфину "
                    "не прошел порог ({} vs {})".format(
                        request_id,
                        answers[0].get_score(),
                        MODEL_CONFIG["relevant_minfin_main_answer_threshold"]
                    ))

    @staticmethod
    def _process_more_answers(
            core_answer: CoreAnswer,
            more_answers: list,
            request_id: str
    ):
        """Обработка дополнительных ответов"""

        more_cube_answers = []
        more_minfin_answers = []

        more_answers_order = ''

        for ind, answer in enumerate(more_answers):
            answer.order = ind

        for answer in more_answers:
            if isinstance(answer, CubeAnswer):
                more_cube_answers.append(answer)
                more_answers_order += '0'
            else:
                more_minfin_answers.append(answer)
                more_answers_order += '1'

        if more_cube_answers:
            core_answer.more_cube_answers = more_cube_answers

        if more_minfin_answers:
            core_answer.more_minfin_answers = more_minfin_answers

        if more_answers_order:
            core_answer.more_answers_order = more_answers_order

        logging.info(
            "Query_ID: {}\tMessage: В смотри также {} "
            "ответ(а, ов) по кубам и {} по Минфину".format(
                request_id,
                len(more_cube_answers),
                len(more_minfin_answers)
            ))

    @staticmethod
    def _manual_cube_classification(user_request: str):
        key_words_to_cube = {
            "основной показатель субъект РФ": "CLDO02",
            "источник финансирование": "FSYR01",
            "оперативный расход": "EXDO01",
            "годовой расход": "EXYR03",
            "оперативный доход": "INDO01",
            "основной характеристика федеральный бюджет": "CLDO01",
            "годовой доход": "INYR03",
            "госдолг РФ": "CLMR02"
        }

        for key, value in key_words_to_cube.items():
            if user_request.startswith(key):
                return True, value

        return False, None


import subprocess
import os
from six import string_types

# SLHA parser
# by Florian Staub (florian.staub@gmail.com)

# ----------------------------------------------------------
# SLHA Class
# ----------------------------------------------------------

class SLHA():
    def __init__(self):
        self.blocks = {}
        self.br = {}
        self.widths = {}
        self.br1L = {}
        self.widths1L = {}
        self.xsections = {}

        self.block_name = None
        self.entries = {}
        self.reading_block = False
        self.reading_decay = False
        self.reading_xsection = False
        self.reading_hb_fermion = False
        self.reading_hb_boson = False
        self.decay1L = False
        self.decay_part = 0

    # return wdith and BR
    def BR(self, init, final):
        # frozenset: make sure that the final states are order-less
        return self.br[init][tuple(sorted(final))]

    def Width(self, pdg):
        return self.widths[pdg]

    def Value(self, block, number):
        '''return value of a parameter defined by block and entry
            or the width or an BR'''
        if block == 'WIDTH':
            return self.widths[number]
        elif block == 'BR':
            return self.br[number[0]][tuple(sorted(number[1]))]
        elif block == 'WIDTH1L':
            return self.widths1L[number]
        elif block == 'BR1L':
            return self.br1L[number[0]][tuple(sorted(number[1]))]
        elif block == 'XSECTION':
            xs = self.xsections[tuple(number)]
            return [[x, xs[x]] for x in xs.keys()]
        else:
            return self.blocks[block.upper()][
                str(number)[1:-1].replace(" ", "")]

    def start_decay(self, li):
        parsed = list(filter(None, li.split(' ')))
        self.decay1L = li.upper().startswith("DECAY1L")
        self.decay_part = int(parsed[1])
        if self.decay1L:
            self.widths1L[self.decay_part] = float(parsed[2])
        else:
            self.widths[self.decay_part] = float(parsed[2])
        self.entries = {}
        self.reading_block, self.reading_decay, self.reading_xsection \
            = False, True, False

    def start_block(self, li):
        self.block_name = (list(filter(None, li.split(' ')))[1]).upper()
        self.entries = {}
        self.reading_block, self.reading_decay, self.reading_xsection \
            = True, False, False
        self.reading_hb_boson = \
            self.block_name in ["HIGGSBOUNDSINPUTHIGGSCOUPLINGSBOSONS",
                                "HIGGSCOUPLINGSBOSONS"]
        self.reading_hb_fermion = \
            self.block_name in ["HIGGSBOUNDSINPUTHIGGSCOUPLINGSFERMIONS",
                                "HIGGSCOUPLINGSFERMIONS"]

    def start_xsection(self, li):
        parsed = list(filter(None, li.split(' ')))
        if "#" in parsed:
            parsed = parsed[:parsed.index("#")]  # remove comments
        self.xs_head = tuple(
            [float(parsed[1]),
             tuple([int(parsed[2]), int(parsed[3])]),
             tuple([int(parsed[-2]), int(parsed[-1])])
             ])
        self.entries = {}
        self.reading_block, self.reading_decay, self.reading_xsection \
            = False, False, True

    def flush(self):
        '''store the information once a block is completely parsed'''
        if len(self.entries) > 0:
            if self.reading_block:
                self.blocks[self.block_name] = self.entries
            if self.reading_decay:
                if self.decay1L:
                    self.br1L[self.decay_part] = self.entries
                else:
                    self.br[self.decay_part] = self.entries
            if self.reading_xsection:
                self.xsections[self.xs_head] = self.entries

# ----------------------------------------------------------
# Reading
# ----------------------------------------------------------

# now the main function to read the SLHA file
def read(file, separator=None, verbose=False):
    spc = SLHA()
    if separator is not None:
        all_files = []
        count = 1

    with open(file) as infile:
        for line in infile:
            li = line.strip().upper()

            if li.startswith("#") or len(li) < 1:
                continue

            if separator is not None:
                if li.startswith(separator):
                    spc.flush()
                    if max(len(spc.blocks.keys()),len(spc.widths.keys())) > 0:
                        all_files.append(spc)
                        # start next point
                        spc = SLHA()
                        count = count + 1
                        if verbose:
                            print("Read spc file:", count)
                        continue

            # New block started
            if li.startswith("BLOCK"):
                spc.flush()  # store information which was read
                spc.start_block(li)
            elif li.startswith("DECAY"):
                spc.flush()  # store information which was read
                spc.start_decay(li)
            elif li.startswith("XSECTION"):
                spc.flush()  # store information which was read
                spc.start_xsection(li)

            # Reading and parsing values
            else:
                parsed = list(filter(None, li.split(' ')))
                if "#" in parsed:
                    parsed = parsed[:parsed.index("#")]  # remove comments
                if spc.reading_block:
                    if spc.reading_hb_fermion:
                        spc.entries[",".join(parsed[3:])] = \
                            [float(parsed[0]), float(parsed[1])]
                    elif spc.reading_hb_boson:
                        spc.entries[",".join(parsed[2:])] = \
                            float(parsed[0])
                    else:
                        # Value might be a string like in SPINFO block
                        try:
                            value = float(parsed[-1])
                        except:
                            value = parsed[-2]
                        spc.entries[",".join(parsed[0:-1])] = value

                if spc.reading_decay:
                    spc.entries[
                        tuple(sorted(eval("[" + ",".join(parsed[2:]) + "]")))
                    ] = float(parsed[0])

                if spc.reading_xsection:
                    spc.entries[
                        tuple(eval("[" + ",".join(parsed[0:-2]) + "]"))
                    ] = float(parsed[-2])

    spc.flush()  # save the very last block in the file

    if verbose:
        print("Read %i blocks and %i decays" % (len(spc.blocks), len(spc.br)))
    if separator is None:
        return spc
    else:
        if len(spc.entries) > 0:
            all_files.append(spc)
        return all_files

# wrapper for faster read-in of multiple files
# squeeze the file (just keeping the necessary entries) to make the reading more efficient
# example: read_small_spc(filename,["# m0","# m12","# relic"],separator="ENDOF")
def read_small(file, entries, sep):
    if entries is None:
        out = read(file, separator=sep)
    else:
        string = "--regexp=\"" + sep + "\" --regexp=\"Block\" "
        for i in entries:
            string = string + "--regexp=\"" + i + "\" "
        if os.path.isfile("temp.spc"):
            subprocess.call("rm temp.spc", shell=True)
        subprocess.call("cat " + file + " | grep -i " + string
                        + " > temp_read_small.spc", shell=True)
        out = read("temp_read_small.spc", separator=sep)
        subprocess.call("rm temp_read_small.spc", shell=True)

    return out

def read_dir(dir, entries=None):
    if os.path.isfile("temp_read_dir.spc"):
        subprocess.call("rm temp_read_dir.spc", shell=True)
#    subprocess.check_call("cat "+dir+"/* > temp_read_dir.spc",shell=True)
    subprocess.check_call("tail -n+1 " + dir + "/* > temp_read_dir.spc",
                          shell=True)
    out = read_small("temp_read_dir.spc", entries, "==>")
    subprocess.call("rm temp_read_dir.spc", shell=True)

    return out

#def read_dir(dir,entries=None):
    #subprocess.call("rm temp_read_dir.spc",shell=True)
    #subprocess.check_call("cat "+dir+"/* > temp_read_dir.spc",shell=True)
    #with open("temp_read_dir.spc") as infile:
     #for line in infile:
       #li=line.strip().upper()
       #if li.startswith("#") or len(li)<1:
          #continue
       #else:
          #file_sep=li[:li.index("#")]
          #break
    #out=read_small("temp_read_dir.spc",entries,file_sep)
    #subprocess.call("rm temp_read_dir.spc",shell=True)

    #return out

# ----------------------------------------------------------
# Writing
# ----------------------------------------------------------

def write(blocks, file):
    with open(file, 'w+') as f:
        for b in blocks:
            write_block_head(b, f)
            write_block_entries(blocks[b], f)

def write_block_entries(values, file):
    for v in values.keys():
        file.write(' %s %10.4e # \n' % (v, float(values[v])))

def write_les_houches(block, values, point, file):
    write_block_head(block, file)
    write_block_numbers(block, values, point, file)

def write_block_head(name, file):
    file.write("Block " + name.upper() + " # \n")

def write_block_numbers(name, values, Variable, file):
    for v in values.keys():
        # if type(values[v]) is string_types:
        if isinstance(values[v], string_types):  # to be 2 and 3 compatible
            if str(eval(values[v]))==values[v]:
                file.write(' %s %s # %s \n'
                           % (v, values[v],
                              name.upper() + "[" + str(v) + "]"))
            else:
                file.write(' %s %10.4e # %s \n'
                           % (v, float(eval(values[v])),
                              name.upper() + "[" + str(v) + "]"))
        elif isinstance(values[v], int):
            file.write(' %s %i # %s \n'
                       % (v, (values[v]), name.upper() + "[" + str(v) + "]"))
        else:
            file.write(' %s %10.4e # %s \n'
                       % (v, float(values[v]),
                          name.upper() + "[" + str(v) + "]"))


"""
This module is for loading data and data management
"""
import torch
from torch.utils.data import Dataset
from torchvision import transforms, datasets
from torch.utils.data import random_split

metaPath = r'../aifood_en/meta/'
imgPath = r'../aifood_en/images/'

# 全局变量

def getSmallSmplFileName():
    """
    获取小数据集名字
    :return:
    """
    f = open(metaPath+r'small_labels_25c.txt','r')
    return f.read().split('\n')

def getBigSmpFileName():
    """
    获取大数据集名字
    :return:
    """
    f = open(metaPath+r'large_labels_75c.txt','r')
    return f.read().split('\n')

# 转换器 把图片转换为可以输入网络的格式
transform = transforms.Compose([
        transforms.Resize((224,224)),
        transforms.ToTensor()]
)

# 转换函数
# def transformFunc(img):
#     # img 为pillow格式
#     im = transform(img)
#     x = Variable(torch.unsqueeze(im, dim=0).float(), requires_grad=False)
#     return x

# 这个没加转换函数，可以直接看图片
imgBrowser = datasets.ImageFolder(root=imgPath,transform=None)

# 这个加了转换函数，用于生成训练集
foodData = datasets.ImageFolder(root=imgPath,transform=transform)

def getFoodDataLoader(batch_size,frac=0.9):
    trainLen = int(len(foodData)* frac)
    train, test = random_split(foodData, [ trainLen, len(foodData)-trainLen])
    return torch.utils.data.DataLoader(train,batch_size=batch_size,shuffle=True),torch.utils.data.DataLoader(test,batch_size=batch_size,shuffle=True)


import math

def calc_circumference(radius):
    circumference = 2 * math.pi * radius

    return circumference

def calc_area(radius):
   area = math.pi * radius * radius

   return area

print("To quit enter a negative value.")
radius = float(input("Give me a radius of your circle:"))

while radius >=0:
    x = calc_circumference(radius)
    y = calc_area(radius)

    print("The circumference is ", x)
    print(" The area is ", y)

    radius = float(input("Give me another radius:"))


import sys
import itertools
start = [[False for i in range(0,1000)] for j in range(0, 1000)]

def set_all(start, line, val):
    foo  = line.split()
    fram = foo[2].split(',')
    to =  foo[4].split(',')
    r1 = range(int(fram[0]), int(to[0]) +1)
    r2 = range(int(fram[1]), int(to[1])+1)
    for (i,j) in itertools.product(r1, r2 ):
        start[i][j] = val

def process(start, line):
    if(line.startswith("turn on")):
        set_all(start, line, True)
    if(line.startswith("turn off")):
        set_all(start, line, False)
    if(line.startswith("toggle")):
        foo  = line.split()
        fram = foo[1].split(',')
        to =  foo[3].split(',')
        r1 = range(int(fram[0]), int(to[0]) +1)
        r2 = range(int(fram[1]), int(to[1])+1)
        for (i,j) in itertools.product(r1, r2 ):
            start[i][j] = not start[i][j]

for line in sys.stdin:
    process(start, line)
print(sum(map(sum, start)))


from collections import deque

from Entidades.Term import TermUnit
from Entidades.Term import Term
from Entidades.Grammar.TextGrammar import TextGrammar
from Entidades.Grammar.Grammar import Grammar
from Services.LexicAnalyzer import LexicAnalyzer

class TableService:
    ErrorString = "Error"

    def __init__(self, grammar):
        self.grammar = grammar
        self.table = None

    def makeTable(self):
        self.table = {}

        for non_terminal in self.grammar.NonTerminals:
            self.table[non_terminal.text] = {Grammar.STREAM_END_UNIT.text: self.ErrorString}

            for terminal in self.grammar.Alphabet:
                self.table[non_terminal.text][terminal.text] = self.ErrorString

    def passToTable(self, i, j, value):
        actualValue = self.table[i][j]

        if(actualValue == self.ErrorString):
            self.table[i][j] = value
        else:
            raise Exception('Ambiguos grammar on '+ i + ' -> ' + j)

    def build_table(self):
        self.makeTable()

        for term in self.grammar.Terms:
            for stream in term.right:
                termToStreamTuple = (term, stream)

                streamFirst = self.first(stream)

                for item in streamFirst:
                    if(item.type is TermUnit.TERMINAL):
                        self.passToTable(term.left, item.text, termToStreamTuple)

                if(TextGrammar.EMPTY_UNIT in streamFirst):
                    if(Grammar.STREAM_END_UNIT in term.follow):
                        self.passToTable(term.left, Grammar.STREAM_END_UNIT.text, termToStreamTuple)

                    for item in term.follow:
                        if(item.type is TermUnit.TERMINAL):
                            self.passToTable(term.left, item.text, termToStreamTuple)

    def apply_follow_third_rule(self):
        """
        A -> Alfa B Beta, B.follow += A.follow
        Alfa is any stream
        Beta is a empty stream
        """
        calls = deque()

        for A in self.grammar.Terms:
            for stream in A.right:
                streamLen = len(stream)

                for index, unit in enumerate(stream):
                    if(unit.type is TermUnit.NONTERMINAL):
                        B = self.get_term(unit.text)

                        if(A is B):
                            continue

                        if(streamLen > index + 1):
                            beta = stream[index + 1::]

                            firstBeta = self.first(beta)

                            if(TextGrammar.EMPTY_UNIT not in firstBeta):
                                continue

                        # B.follow |= A
                        calls.append((B, A))

        while(len(calls) is not 0):
            tupleBA = calls.popleft()
            B, A = tupleBA

            AinStack = next((x for x in calls if x[0] is A), None)

            if(AinStack):
                calls.append(tupleBA)
                continue

            # print(B.left,"___", A.left)

            B.follow |= A.follow

    def apply_follow_second_rule(self):
        """
        A -> Alfa B Beta, B.follow += first(Beta) - {ɛ}
        Alfa is any stream
        Beta is a not empty stream
        """
        for term in self.grammar.Terms:
            for stream in term.right:
                streamLen = len(stream)

                for index, unit in enumerate(stream):
                    if(unit.type is TermUnit.NONTERMINAL):
                        termB = self.get_term(unit.text)

                        if(streamLen > index + 1):
                            beta = stream[index + 1::]
                            firstBeta = self.first(beta)

                            termB.follow |= set(firstBeta) - {self.grammar.textGrammar.EMPTY_UNIT}

    def apply_follow_first_rule(self):
        """
        S.follow = $
        """
        self.grammar.StartSimbol.follow.add(Grammar.STREAM_END_UNIT)

    def build_follow(self):
        """
        Routine that creates follow sets of the given grammar
        """
        self.apply_follow_first_rule()
        self.apply_follow_second_rule()
        self.apply_follow_third_rule()

    def first_of_non_terminal(self, term):
        if(len(term.first) is not 0):
            return term.first

        first = term.first
        for stream in term.right:
            if(len(stream) is 1 and stream[0] is self.grammar.textGrammar.EMPTY_UNIT):
                first |= {self.grammar.textGrammar.EMPTY_UNIT}
            else:
                first |= self.first(stream) - {self.grammar.textGrammar.EMPTY_UNIT}

        term.first = first

        return first

    def term_has_empty(self, term):
        for stream in term.right:
            for item in stream:
                if(item is TextGrammar.EMPTY_UNIT):
                    return True

        return False

    def get_term(self, termString):
        for i in self.grammar.Terms:
            if(i.left == termString):
                return i

    def first_of_stream(self, stream):
        retorno = set()

        for index, item in enumerate(stream):
            if(item.type is TermUnit.NONTERMINAL):
                term = self.get_term(item.text)

                retorno |= self.first(term)

                if(self.term_has_empty(term)):
                    continue
                else:
                    break

            elif(index is 0 and item.type is TermUnit.TERMINAL or item is TextGrammar.EMPTY_UNIT):
                retorno.add(item)
                break

        return retorno

    def first(self, p) :
        p_type = type(p)
        retorno = None

        if(p_type is list):
            retorno = self.first_of_stream(p)

        elif(p_type is TermUnit):
            term = self.get_term(p)
            retorno = self.first_of_non_terminal(term)

        elif(p_type is Term):
            retorno = self.first_of_non_terminal(p)

        return retorno

    def build_first(self):
        for term in self.grammar.Terms:
            self.first(term)

    def compileGrammar(self):
        if(self.table is None):
            self.build_first()
            self.build_follow()
            self.build_table()

    def compile(self, text):
        self.compileGrammar()
        historic = ''
        lxa = LexicAnalyzer(text, self.grammar)

        stack = [Grammar.STREAM_END_UNIT, self.grammar.StartSimbolUnit]

        # while(lxa.isNotDone()):
        #     print(lxa.getToken())

        current = lxa.getToken()
        while( len(stack) ):#current.unit.type is not TermUnit.STREAM_END):
            hline = str(current.value).ljust(20) + " " + str(stack) + '\n'
            historic += hline

            top = stack.pop()
            # print(hline)

            if(top.type is TermUnit.TERMINAL or top.type is TermUnit.STREAM_END):
                if(top.text == current.value):
                    current = lxa.getToken()

                elif(top.text == 'ide'):
                    current = lxa.getToken()

                elif(top.text == 'num'):
                    current = lxa.getToken()

                else:
                    raise Exception("Error on " + current.value + " at:")

            else:
                tupleCell = self.table[top.text][current.unit.text]

                if(tupleCell != self.ErrorString):
                    for term in tupleCell[1][::-1]:
                        if(term.type is not TermUnit.EMPTY):
                            stack.append(term)

                elif(current.unit.type is TermUnit.STREAM_END):
                    raise Exception("Error: invalid file end")

                else:
                    raise Exception("Error on " + current.value + " at:")

        # print("Compiling succes for entry:\n"+ text)
        return lxa.tokens, historic

Python/Analisador Sintático Descendente Tabular/Services/GrammarTableService.py


# Partition Labels
'''
Intuition
Let's try to repeatedly choose the smallest left-justified partition. Consider the first label, say it's 'a'. The first partition must include it, and also the last occurrence of 'a'. However, between those two occurrences of 'a', there could be other labels that make the minimum size of this partition bigger. For example, in "abccaddbeffe", the minimum first partition is "abccaddb". This gives us the idea for the algorithm: For each letter encountered, process the last occurrence of that letter, extending the current partition [anchor, j] appropriately.

Algorithm
We need an array last[char] -> index of S where char occurs last. Then, let anchor and j be the start and end of the current partition. If we are at a label that occurs last at some index after j, we'll extend the partition j = last[c]. If we are at the end of the partition (i == j) then we'll append a partition size to our answer, and set the start of our new partition to i+1.
'''

class Solution:
    def partitionLabels(self, S: str) -> List[int]:
        last = {c: i for i,c in enumerate(S)}
        start = end = 0
        ans = []
        for i,c in enumerate(S):
            end = max(end, last[c])
            if i == end:
                ans.append(i-start+1)
                start = i + 1
        return ans


#Write a Python program to convert temperatures to and from Celsius, Fahrenheit.
# [Formula: c/5 = f-32/9 [ where c = temperature in Celsius and f = temperature in Fahrenheit ] Expected
# Output : 60°C is 140 in Fahrenheit 45°F is 7 in Celsius
t=int(input("Enter Tempreture::"))
u=input("Enter C for celsis and F for Fahrenheit you want to convert::").upper()
if u=='C':
    c=float((t*1.8)+32)
    print("Tempreture in celcius :: %f"%c)
elif u=='F':
    f=float((t-32)*5/9)
    print("Tempreture in Fahrenheit is ::%f"%f)


import requests
from bs4 import BeautifulSoup as BS
from pprint import pprint

def parse_cars():
    url = 'https://www.mashina.kg/search/all/'
    response = requests.get(url, timeout=15)
    print(response)
    print(response.status_code == 200)
    # print(response.text[:200])
    cars = []
    soup = BS(response.text, "html.parser")
    table = soup.find('div', class_="table-view-list")
    obyavlenia = table.find_all('a')
    for data in obyavlenia:
        car = {}
        car['title'] = data.find('div', class_="title").text.replace('\n', '').strip()
        car['price'] = data.find('div', class_="block price").find("strong").text.replace('\n', '').replace('$ ', '').replace(' ', '')
        car['info'] = ''
        cars.append(car)

    return cars

# x = '       \n\n\n     hello             world        \n     '
# print(
#     ' '.join(
#             x.strip().replace('\n', '').split()
#         )
#     )


from distutils import extension
import os
import shutil

from_dir = "/Users/satya/Downloads"

to_dir = "/Users/satya/Downloads"

list_of_files = os.listdir(from_dir)
print(list_of_files)

for file_name in list_of_files :
    name,ext = os.path.splitext(file_name)
    print(name)
    print(ext)
    if ext == ' ' :
        continue
    if ext in ['.doc', '.docx', '.rtf', '.txt', '.pdf'] :
        path1 = from_dir + '/' + file_name
        path2 = to_dir + '/' + 'Document_Files'
        path3 = to_dir + '/' + 'Document_Files' + '/' + file_name
        print('moving from', path1)
        print('moving to', path3)
        if os.path.exists(path2) :
            print("Moving", file_name, "to", path2, '  .....')
            shutil.move(path1, path2)
        else :
            os.mkdir(path2)
            print("Moving", file_name, "to", path2, '  ......')
            shutil.move(path1, path2)


"""
 This exception is thrown if the emulator tries to read a value bigger than
 the allowed maximum
"""
class InvalidReadSize(Exception):
    def __init__(self, size: int):
        self.offset = size
        self.message = f"Cannot read {size} bytes"
        super().__init__(self.message)

class AddressOutOfRange(Exception):
    def __init__(self, address: int, max_size: int):
        self.address = address
        self.message = f"Address {address} is out of range. Maximum address is {max_size}"
        super().__init__(self.message)

class InvalidCr0Combination(Exception):
    def __init__(self, msg: str):
        self.message = "Invalid Combination for CR0 ".append(msg)
        super().__init__(self.message)

class AccessToReservedCr0Field(Exception):
    def __init__(self, field: int):
        self.field = field
        self.message = f"Bit field at idx {field} is reserved and shall not be modified"
        super().__init__(self.message)

class InvalidMsrIndex(Exception):
    def __init__(self, idx: int):
        self.idx = idx
        self.message = f"Invalid MSR index: {idx}"
        super().__init__(self.message)

class AccessToReservedMsrIndex(Exception):
    def __init__(self, msr_idx: int, bit_field: int):
        self.msr_idx = msr_idx
        self.bit_field = bit_field
        self.message = f"Cannot change bit number {bit_field} of {msr_idx} MSR"
        super().__init__(self.message)

class InvalidEferCombination(Exception):
    def __init__(self, field_name: str, value: bool, ext: int):
        self.field_name = field_name
        self.value = value
        self.message = f"""Invalid combination for field {field_name}, value {value}.
            Conflicts with: {ext:#0{4}x}"""

class InvalidEferAccess(Exception):
    def __init__(self, field_name: str, value: bool):
        self.message = f"Cannot change field {field_name} to {value}"
        super().__init__(self.message)

class PageFault(Exception):
    def __init__(self, is_minor: bool, virtual_address: int, address_length: int):
        self.is_minor = is_minor
        self.virtual_address = virtual_address
        pf_type_str = "minor" if is_minor else "major"
        self.message = f"A {pf_type_str} page-fault occured at {virtual_address:#0{address_length}x}"
        super().__init__(self.message)