import torch
from torch import nn
from tqdm.auto import tqdm
from torchvision import transforms
from torchvision.datasets import MNIST # Training dataset
from torchvision.utils import make_grid
from torch.utils.data import DataLoader
import matplotlib.pyplot as plt
torch.manual_seed(0)

def show_tensor_images(image_tensor, num_images=25, size=(1, 28, 28)):
    '''
    Function for visualizing images: Given a tensor of images, number of images, and
    size per image, plots and prints the images in a uniform grid.
    '''
    image_unflat = image_tensor.detach().cpu().view(-1, *size)
    image_grid = make_grid(image_unflat[:num_images], nrow=5)
    plt.imshow(image_grid.permute(1, 2, 0).squeeze())
    plt.show()

# UNQ_C1 (UNIQUE CELL IDENTIFIER, DO NOT EDIT)
# GRADED FUNCTION: get_generator_block
def get_generator_block(input_dim, output_dim):
    '''
    Function for returning a block of the generator's neural network
    given input and output dimensions.
    Parameters:
        input_dim: the dimension of the input vector, a scalar
        output_dim: the dimension of the output vector, a scalar
    Returns:
        a generator neural network layer, with a linear transformation
          followed by a batch normalization and then a relu activation
    '''
    return nn.Sequential(
        # Hint: Replace all of the "None" with the appropriate dimensions.
        # The documentation may be useful if you're less familiar with PyTorch:
        # https://pytorch.org/docs/stable/nn.html.
        #### START CODE HERE ####
        nn.Linear(input_dim,output_dim),
        nn.BatchNorm1d(output_dim),
        nn.ReLU(inplace=True),
        #### END CODE HERE ####
    )

# Verify the generator block function
def test_gen_block(in_features, out_features, num_test=1000):
    block = get_generator_block(in_features, out_features)

    # Check the three parts
    assert len(block) == 3
    assert type(block[0]) == nn.Linear
    assert type(block[1]) == nn.BatchNorm1d
    assert type(block[2]) == nn.ReLU

    # Check the output shape
    test_input = torch.randn(num_test, in_features)
    test_output = block(test_input)
    assert tuple(test_output.shape) == (num_test, out_features)
    assert test_output.std() > 0.55
    assert test_output.std() < 0.65

test_gen_block(25, 12)
test_gen_block(15, 28)
print("Success!")

# UNQ_C2 (UNIQUE CELL IDENTIFIER, DO NOT EDIT)
# GRADED FUNCTION: Generator
class Generator(nn.Module):
    '''
    Generator Class
    Values:
        z_dim: the dimension of the noise vector, a scalar
        im_dim: the dimension of the images, fitted for the dataset used, a scalar
          (MNIST images are 28 x 28 = 784 so that is your default)
        hidden_dim: the inner dimension, a scalar
    '''
    def __init__(self, z_dim=10, im_dim=784, hidden_dim=128):
        super(Generator, self).__init__()
        # Build the neural network
        self.gen = nn.Sequential(
            get_generator_block(z_dim, hidden_dim),
            get_generator_block(hidden_dim, hidden_dim * 2),
            get_generator_block(hidden_dim * 2, hidden_dim * 4),
            get_generator_block(hidden_dim * 4, hidden_dim * 8),
            # There is a dropdown with hints if you need them!
            #### START CODE HERE ####
            nn.Linear(hidden_dim*8,im_dim),
            nn.Sigmoid()
            #### END CODE HERE ####
        )
    def forward(self, noise):
        '''
        Function for completing a forward pass of the generator: Given a noise tensor,
        returns generated images.
        Parameters:
            noise: a noise tensor with dimensions (n_samples, z_dim)
        '''
        return self.gen(noise)

    # Needed for grading
    def get_gen(self):
        '''
        Returns:
            the sequential model
        '''
        return self.gen

# Verify the generator class
def test_generator(z_dim, im_dim, hidden_dim, num_test=10000):
    gen = Generator(z_dim, im_dim, hidden_dim).get_gen()

    # Check there are six modules in the sequential part
    assert len(gen) == 6
    test_input = torch.randn(num_test, z_dim)
    test_output = gen(test_input)

    # Check that the output shape is correct
    assert tuple(test_output.shape) == (num_test, im_dim)
    assert test_output.max() < 1, "Make sure to use a sigmoid"
    assert test_output.min() > 0, "Make sure to use a sigmoid"
    assert test_output.min() < 0.5, "Don't use a block in your solution"
    assert test_output.std() > 0.05, "Don't use batchnorm here"
    assert test_output.std() < 0.15, "Don't use batchnorm here"

test_generator(5, 10, 20)
test_generator(20, 8, 24)
print("Success!")

# UNQ_C3 (UNIQUE CELL IDENTIFIER, DO NOT EDIT)
# GRADED FUNCTION: get_noise
def get_noise(n_samples, z_dim, device='cpu'):
    '''
    Function for creating noise vectors: Given the dimensions (n_samples, z_dim),
    creates a tensor of that shape filled with random numbers from the normal distribution.
    Parameters:
        n_samples: the number of samples to generate, a scalar
        z_dim: the dimension of the noise vector, a scalar
        device: the device type
    '''
    # NOTE: To use this on GPU with device='cuda', make sure to pass the device
    # argument to the function you use to generate the noise.
    #### START CODE HERE ####
    return torch.randn(n_samples,z_dim,device=device)
    #### END CODE HERE ####

# Verify the noise vector function
def test_get_noise(n_samples, z_dim, device='cpu'):
    noise = get_noise(n_samples, z_dim, device)

    # Make sure a normal distribution was used
    assert tuple(noise.shape) == (n_samples, z_dim)
    assert torch.abs(noise.std() - torch.tensor(1.0)) < 0.01
    assert str(noise.device).startswith(device)

test_get_noise(1000, 100, 'cpu')
if torch.cuda.is_available():
    test_get_noise(1000, 32, 'cuda')
print("Success!")

# UNQ_C4 (UNIQUE CELL IDENTIFIER, DO NOT EDIT)
# GRADED FUNCTION: get_discriminator_block
def get_discriminator_block(input_dim, output_dim):
    '''
    Discriminator Block
    Function for returning a neural network of the discriminator given input and output dimensions.
    Parameters:
        input_dim: the dimension of the input vector, a scalar
        output_dim: the dimension of the output vector, a scalar
    Returns:
        a discriminator neural network layer, with a linear transformation
          followed by an nn.LeakyReLU activation with negative slope of 0.2
          (https://pytorch.org/docs/master/generated/torch.nn.LeakyReLU.html)
    '''
    return nn.Sequential(
        #### START CODE HERE ####
        nn.Linear(input_dim,output_dim),
        nn.LeakyReLU(negative_slope=0.2)
        #### END CODE HERE ####
    )

# Verify the discriminator block function
def test_disc_block(in_features, out_features, num_test=10000):
    block = get_discriminator_block(in_features, out_features)

    # Check there are two parts
    assert len(block) == 2
    test_input = torch.randn(num_test, in_features)
    test_output = block(test_input)

    # Check that the shape is right
    assert tuple(test_output.shape) == (num_test, out_features)

    # Check that the LeakyReLU slope is about 0.2
    assert -test_output.min() / test_output.max() > 0.1
    assert -test_output.min() / test_output.max() < 0.3
    assert test_output.std() > 0.3
    assert test_output.std() < 0.5

test_disc_block(25, 12)
test_disc_block(15, 28)
print("Success!")

# UNQ_C5 (UNIQUE CELL IDENTIFIER, DO NOT EDIT)
# GRADED FUNCTION: Discriminator
class Discriminator(nn.Module):
    '''
    Discriminator Class
    Values:
        im_dim: the dimension of the images, fitted for the dataset used, a scalar
            (MNIST images are 28x28 = 784 so that is your default)
        hidden_dim: the inner dimension, a scalar
    '''
    def __init__(self, im_dim=784, hidden_dim=128):
        super(Discriminator, self).__init__()
        self.disc = nn.Sequential(
            get_discriminator_block(im_dim, hidden_dim * 4),
            get_discriminator_block(hidden_dim * 4, hidden_dim * 2),
            get_discriminator_block(hidden_dim * 2, hidden_dim),
            # Hint: You want to transform the final output into a single value,
            #       so add one more linear map.
            #### START CODE HERE ####
            nn.Linear(hidden_dim,1)
            #### END CODE HERE ####
        )

    def forward(self, image):
        '''
        Function for completing a forward pass of the discriminator: Given an image tensor,
        returns a 1-dimension tensor representing fake/real.
        Parameters:
            image: a flattened image tensor with dimension (im_dim)
        '''
        return self.disc(image)

    # Needed for grading
    def get_disc(self):
        '''
        Returns:
            the sequential model
        '''
        return self.disc

# Verify the discriminator class
def test_discriminator(z_dim, hidden_dim, num_test=100):

    disc = Discriminator(z_dim, hidden_dim).get_disc()

    # Check there are three parts
    assert len(disc) == 4

    # Check the linear layer is correct
    test_input = torch.randn(num_test, z_dim)
    test_output = disc(test_input)
    assert tuple(test_output.shape) == (num_test, 1)

    # Don't use a block
    assert not isinstance(disc[-1], nn.Sequential)

test_discriminator(5, 10)
test_discriminator(20, 8)
print("Success!")

#Training

# Set your parameters
criterion = nn.BCEWithLogitsLoss()
n_epochs = 200
z_dim = 64
display_step = 500
batch_size = 128
lr = 0.00001

# Load MNIST dataset as tensors
dataloader = DataLoader(
    MNIST('.', download=False, transform=transforms.ToTensor()),
    batch_size=batch_size,
    shuffle=True)

### DO NOT EDIT ###
device = 'cuda'

gen = Generator(z_dim).to(device)
gen_opt = torch.optim.Adam(gen.parameters(), lr=lr)
disc = Discriminator().to(device)
disc_opt = torch.optim.Adam(disc.parameters(), lr=lr)

# UNQ_C6 (UNIQUE CELL IDENTIFIER, DO NOT EDIT)
# GRADED FUNCTION: get_disc_loss
def get_disc_loss(gen, disc, criterion, real, num_images, z_dim, device):
    '''
    Return the loss of the discriminator given inputs.
    Parameters:
        gen: the generator model, which returns an image given z-dimensional noise
        disc: the discriminator model, which returns a single-dimensional prediction of real/fake
        criterion: the loss function, which should be used to compare
               the discriminator's predictions to the ground truth reality of the images
               (e.g. fake = 0, real = 1)
        real: a batch of real images
        num_images: the number of images the generator should produce,
                which is also the length of the real images
        z_dim: the dimension of the noise vector, a scalar
        device: the device type
    Returns:
        disc_loss: a torch scalar loss value for the current batch
    '''
    #     These are the steps you will need to complete:
    #       1) Create noise vectors and generate a batch (num_images) of fake images.
    #            Make sure to pass the device argument to the noise.
    #       2) Get the discriminator's prediction of the fake image
    #            and calculate the loss. Don't forget to detach the generator!
    #            (Remember the loss function you set earlier -- criterion. You need a
    #            'ground truth' tensor in order to calculate the loss.
    #            For example, a ground truth tensor for a fake image is all zeros.)
    #       3) Get the discriminator's prediction of the real image and calculate the loss.
    #       4) Calculate the discriminator's loss by averaging the real and fake loss
    #            and set it to disc_loss.
    #     Note: Please do not use concatenation in your solution. The tests are being updated to
    #           support this, but for now, average the two losses as described in step (4).
    #     *Important*: You should NOT write your own loss function here - use criterion(pred, true)!
    #### START CODE HERE ####
    noise=get_noise(num_images, z_dim, device=device)
    fake_img=gen(noise)
    pred_f=disc(fake_img.detach())
    true_f=torch.zeros_like(torch.empty(num_images, 1),device=device)
    f_loss=criterion(pred_f,true_f)
    pred_r=disc(real)
    true_r=torch.ones_like(torch.empty(num_images, 1),device=device)
    r_loss=criterion(pred_r,true_r)
    disc_loss=(f_loss+r_loss)/2

    #### END CODE HERE ####

    return disc_loss

def test_disc_reasonable(num_images=10):
    # Don't use explicit casts to cuda - use the device argument
    import inspect, re
    lines = inspect.getsource(get_disc_loss)
    assert (re.search(r"to\(.cuda.\)", lines)) is None
    assert (re.search(r"\.cuda\(\)", lines)) is None

    z_dim = 64
    gen = torch.zeros_like
    disc = lambda x: x.mean(1)[:, None]
    criterion = torch.mul # Multiply
    real = torch.ones(num_images, z_dim)
    disc_loss = get_disc_loss(gen, disc, criterion, real, num_images, z_dim, 'cpu')
    assert torch.all(torch.abs(disc_loss.mean() - 0.5) < 1e-5)

    gen = torch.ones_like
    criterion = torch.mul # Multiply
    real = torch.zeros(num_images, z_dim)
    assert torch.all(torch.abs(get_disc_loss(gen, disc, criterion, real, num_images, z_dim, 'cpu')) < 1e-5)

    gen = lambda x: torch.ones(num_images, 10)
    disc = lambda x: x.mean(1)[:, None] + 10
    criterion = torch.mul # Multiply
    real = torch.zeros(num_images, 10)
    assert torch.all(torch.abs(get_disc_loss(gen, disc, criterion, real, num_images, z_dim, 'cpu').mean() - 5) < 1e-5)

    gen = torch.ones_like
    disc = nn.Linear(64, 1, bias=False)
    real = torch.ones(num_images, 64) * 0.5
    disc.weight.data = torch.ones_like(disc.weight.data) * 0.5
    disc_opt = torch.optim.Adam(disc.parameters(), lr=lr)
    criterion = lambda x, y: torch.sum(x) + torch.sum(y)
    disc_loss = get_disc_loss(gen, disc, criterion, real, num_images, z_dim, 'cpu').mean()
    disc_loss.backward()
    assert torch.isclose(torch.abs(disc.weight.grad.mean() - 11.25), torch.tensor(3.75))

def test_disc_loss(max_tests = 10):
    z_dim = 64
    gen = Generator(z_dim).to(device)
    gen_opt = torch.optim.Adam(gen.parameters(), lr=lr)
    disc = Discriminator().to(device)
    disc_opt = torch.optim.Adam(disc.parameters(), lr=lr)
    num_steps = 0
    for real, _ in dataloader:
        cur_batch_size = len(real)
        real = real.view(cur_batch_size, -1).to(device)

        ### Update discriminator ###
        # Zero out the gradient before backpropagation
        disc_opt.zero_grad()

        # Calculate discriminator loss
        disc_loss = get_disc_loss(gen, disc, criterion, real, cur_batch_size, z_dim, device)
        assert (disc_loss - 0.68).abs() < 0.05

        # Update gradients
        disc_loss.backward(retain_graph=True)

        # Check that they detached correctly
        assert gen.gen[0][0].weight.grad is None

        # Update optimizer
        old_weight = disc.disc[0][0].weight.data.clone()
        disc_opt.step()
        new_weight = disc.disc[0][0].weight.data

        # Check that some discriminator weights changed
        assert not torch.all(torch.eq(old_weight, new_weight))
        num_steps += 1
        if num_steps >= max_tests:
            break

test_disc_reasonable()
test_disc_loss()
print("Success!")

# UNQ_C7 (UNIQUE CELL IDENTIFIER, DO NOT EDIT)
# GRADED FUNCTION: get_gen_loss
def get_gen_loss(gen, disc, criterion, num_images, z_dim, device):
    '''
    Return the loss of the generator given inputs.
    Parameters:
        gen: the generator model, which returns an image given z-dimensional noise
        disc: the discriminator model, which returns a single-dimensional prediction of real/fake
        criterion: the loss function, which should be used to compare
               the discriminator's predictions to the ground truth reality of the images
               (e.g. fake = 0, real = 1)
        num_images: the number of images the generator should produce,
                which is also the length of the real images
        z_dim: the dimension of the noise vector, a scalar
        device: the device type
    Returns:
        gen_loss: a torch scalar loss value for the current batch
    '''
    #     These are the steps you will need to complete:
    #       1) Create noise vectors and generate a batch of fake images.
    #           Remember to pass the device argument to the get_noise function.
    #       2) Get the discriminator's prediction of the fake image.
    #       3) Calculate the generator's loss. Remember the generator wants
    #          the discriminator to think that its fake images are real
    #     *Important*: You should NOT write your own loss function here - use criterion(pred, true)!

    #### START CODE HERE ####
    noise=get_noise(num_images, z_dim, device=device)
    fake_img=gen(noise)
    pred_f=disc(fake_img)
    true_f=torch.ones_like(torch.empty(num_images, 1),device=device)
    gen_loss=criterion(pred_f,true_f)
    #### END CODE HERE ####
    return gen_loss

def test_gen_reasonable(num_images=10):
        # Don't use explicit casts to cuda - use the device argument
    import inspect, re
    lines = inspect.getsource(get_gen_loss)
    assert (re.search(r"to\(.cuda.\)", lines)) is None
    assert (re.search(r"\.cuda\(\)", lines)) is None

    z_dim = 64
    gen = torch.zeros_like
    disc = nn.Identity()
    criterion = torch.mul # Multiply
    gen_loss_tensor = get_gen_loss(gen, disc, criterion, num_images, z_dim, 'cpu')
    assert torch.all(torch.abs(gen_loss_tensor) < 1e-5)
    #Verify shape. Related to gen_noise parametrization
    assert tuple(gen_loss_tensor.shape) == (num_images, z_dim)

    gen = torch.ones_like
    disc = nn.Identity()
    criterion = torch.mul # Multiply
    real = torch.zeros(num_images, 1)
    gen_loss_tensor = get_gen_loss(gen, disc, criterion, num_images, z_dim, 'cpu')
    assert torch.all(torch.abs(gen_loss_tensor - 1) < 1e-5)
    #Verify shape. Related to gen_noise parametrization
    assert tuple(gen_loss_tensor.shape) == (num_images, z_dim)

def test_gen_loss(num_images):
    z_dim = 64
    gen = Generator(z_dim).to(device)
    gen_opt = torch.optim.Adam(gen.parameters(), lr=lr)
    disc = Discriminator().to(device)
    disc_opt = torch.optim.Adam(disc.parameters(), lr=lr)

    gen_loss = get_gen_loss(gen, disc, criterion, num_images, z_dim, device)

    # Check that the loss is reasonable
    assert (gen_loss - 0.7).abs() < 0.1
    gen_loss.backward()
    old_weight = gen.gen[0][0].weight.clone()
    gen_opt.step()
    new_weight = gen.gen[0][0].weight
    assert not torch.all(torch.eq(old_weight, new_weight))

test_gen_reasonable(10)
test_gen_loss(18)
print("Success!")

# UNQ_C8 (UNIQUE CELL IDENTIFIER, DO NOT EDIT)
# GRADED FUNCTION:

cur_step = 0
mean_generator_loss = 0
mean_discriminator_loss = 0
test_generator = True # Whether the generator should be tested
gen_loss = False
error = False
for epoch in range(n_epochs):

    # Dataloader returns the batches
    for real, _ in tqdm(dataloader):
        cur_batch_size = len(real)

        # Flatten the batch of real images from the dataset
        real = real.view(cur_batch_size, -1).to(device)

        ### Update discriminator ###
        # Zero out the gradients before backpropagation
        disc_opt.zero_grad()

        # Calculate discriminator loss
        disc_loss = get_disc_loss(gen, disc, criterion, real, cur_batch_size, z_dim, device)

        # Update gradients
        disc_loss.backward(retain_graph=True)

        # Update optimizer
        disc_opt.step()

        # For testing purposes, to keep track of the generator weights
        if test_generator:
            old_generator_weights = gen.gen[0][0].weight.detach().clone()

        ### Update generator ###
        #     Hint: This code will look a lot like the discriminator updates!
        #     These are the steps you will need to complete:
        #       1) Zero out the gradients.
        #       2) Calculate the generator loss, assigning it to gen_loss.
        #       3) Backprop through the generator: update the gradients and optimizer.
        #### START CODE HERE ####
        gen_opt.zero_grad()
        gen_loss=get_gen_loss(gen, disc, criterion,cur_batch_size, z_dim, device)
        gen_loss.backward(retain_graph=True)
        gen_opt.step()
        #### END CODE HERE ####

        # For testing purposes, to check that your code changes the generator weights
        if test_generator:
            try:
                assert lr > 0.0000002 or (gen.gen[0][0].weight.grad.abs().max() < 0.0005 and epoch == 0)
                assert torch.any(gen.gen[0][0].weight.detach().clone() != old_generator_weights)
            except:
                error = True
                print("Runtime tests have failed")

        # Keep track of the average discriminator loss
        mean_discriminator_loss += disc_loss.item() / display_step

        # Keep track of the average generator loss
        mean_generator_loss += gen_loss.item() / display_step

        ### Visualization code ###
        if cur_step % display_step == 0 and cur_step > 0:
            print(f"Epoch {epoch}, step {cur_step}: Generator loss: {mean_generator_loss}, discriminator loss: {mean_discriminator_loss}")
            fake_noise = get_noise(cur_batch_size, z_dim, device=device)
            fake = gen(fake_noise)
            show_tensor_images(fake)
            show_tensor_images(real)
            mean_generator_loss = 0
            mean_discriminator_loss = 0
        cur_step += 1


import pickle
import pandas as pd
import seaborn as sns
import numpy as np
import matplotlib.pyplot as plt

from scipy import stats
from scipy.stats import norm
from sklearn import preprocessing

import warnings

warnings.filterwarnings('ignore')

print("this is cleaning work")
data_train = pd.read_csv("data/train.csv")
data_test = pd.read_csv("data/test.csv")

# data_train['SalePrice'].describe()
# # count      1460.000000
# # mean     180921.195890
# # std       79442.502883
# # min       34900.000000
# # 25%      129975.000000
# # 50%      163000.000000
# # 75%      214000.000000
# # max      755000.000000
#
#
# data_train['SalePrice'].skew()
# # 6.536281860064529
# data_train['SalePrice'].kurt()
# # 1.8828757597682129

# corrmat = data_train.corr()
# # f, ax = plt.subplots(figsize=(20, 9))
# # sns.heatmap(corrmat, vmax=0.8, square=True)
#
# k  = 20 # 关系矩阵中将显示20个特征
# cols = corrmat.nlargest(k, 'SalePrice')['SalePrice'].index
# cm = np.corrcoef(data_train[cols].values.T)
# sns.set(font_scale=1)
# hm = sns.heatmap(cm,
#                  cbar=True,
#                  annot=True,
#                  square=True,
#                  fmt='.2f',
#                  annot_kws={'size': 6},
#                  yticklabels=cols.values,
#                  xticklabels=cols.values
#                  )
# plt.show()

# #
# # temp_key = 'Alley'
# # temp = data_train['Alley'].value_counts()
#
# temp_key = 'MSZoning'

#
df = data_train[
    ['OverallQual', 'GrLivArea', 'TotalBsmtSF', 'GarageArea', '1stFlrSF', 'FullBath', 'TotRmsAbvGrd', 'YearRemodAdd',
     'SalePrice']]

df_test = data_test[
    ['OverallQual', 'GrLivArea', 'TotalBsmtSF', 'GarageArea', '1stFlrSF', 'FullBath', 'TotRmsAbvGrd', 'YearRemodAdd']]

f_names = ['YearBuilt', 'CentralAir', 'Neighborhood', 'RoofMatl', 'HouseStyle', 'KitchenQual', 'SaleCondition',
           'SaleType']

# # temp_key = 'Alley'
# # temp = data_train['Alley'].value_counts()
data_test['KitchenQual'].value_counts()
for x in f_names:
    label = preprocessing.LabelEncoder()

    print("x= ", x, "\ttypeA: ", type(data_train[x]), "\ttypeB: ",type(data_test[x]))
    # temp_df = pd.merge(data_train[x], data_test[x])
    # label.fit_transform(temp_df)

    data_train[x].fillna(data_train[x].mode().iloc[0])
    data_test[x].fillna(data_test[x].mode().iloc[0])

    df[x] = label.fit_transform(data_train[x])
    df_test[x] = label.fit_transform(data_test[x])

# try:
#     data_train[x].fillna(data_train[x].mode())
#     d = label.fit_transform(data_train[x])
# except Exception as e:
#     print("x=", x, "\t", e)

# 归一化

df.to_csv('./cleaning.csv', index=False)
df_test.to_csv('./cleaning_test.csv', index=False)


from __future__ import annotations

from typing import Any

import structlog

from scenario_player import runner as scenario_runner
from scenario_player.exceptions import ScenarioAssertionError, ScenarioError
from scenario_player.tasks.api_base import RESTAPIActionTask
from scenario_player.tasks.base import Task

log = structlog.get_logger(__name__)

class AssertPFSRoutesTask(RESTAPIActionTask):
    """
    Assert PFS routes task

    Example usages:

        # Check that PFS response contains 2 routes
        assert_pfs_routes: {from: 0, to: 1, amount: 100, expected_paths: 2}

        # Check that PFS response contains 3 of 3 requested routes
        assert_pfs_routes: {source: 0, target: 1, amount: 100, max_paths: 3, expected_paths: 3}

        Default of `max_paths` is 5
    """

    _name = "assert_pfs_routes"
    _method = "post"
    _url_template = "{pfs_url}/api/v1/{token_network_address}/paths"

    @property
    def _request_params(self):
        source = self._config["from"]
        if isinstance(source, str) and len(source) == 42:
            source_address = source
        else:
            source_address = self._runner.get_node_address(source)

        target = self._config["to"]
        if isinstance(target, str) and len(target) == 42:
            target_address = target
        else:
            target_address = self._runner.get_node_address(target)

        amount = int(self._config["amount"])
        max_paths = int(self._config.get("max_paths", 5))

        params = {
            "from": source_address,
            "to": target_address,
            "value": amount,
            "max_paths": max_paths,
        }
        return params

    @property
    def _url_params(self):
        pfs_url = self._runner.yaml.settings.services.pfs.url
        if not pfs_url:
            raise ScenarioError("PFS tasks require settings.services.pfs.url to be set.")

        params = dict(pfs_url=pfs_url, token_network_address=self._runner.token_network_address)
        return params

    def _process_response(self, response_dict: dict):
        paths = response_dict.get("result")
        if paths is None:
            raise ScenarioAssertionError("No 'result' key in result from PFS")

        num_paths = len(paths)
        exptected_paths = int(self._config["expected_paths"])
        if num_paths != exptected_paths:
            log.debug("Received paths", paths=paths)
            raise ScenarioAssertionError(
                f"Expected {exptected_paths} paths, but PFS returned {num_paths} paths."
            )

class AssertPFSHistoryTask(RESTAPIActionTask):
    """
    Assert PFS history task

    Example usages:

        # 4 requests where made from source node 0
        assert_pfs_history: {source: 0, request_count: 4}

        # 4 requests where made from source node 0 to target node 1
        assert_pfs_history: {source: 0, target: 1, request_count: 4}

        # 4 requests where made from source node 0 to target node 1 and 3 routes each have been
        # returned
        assert_pfs_history: {source: 0, target: 1, request_count: 4, routes_count: 3}

        # 4 requests where made from source node 0 to target node 1 and the specified number of
        # routes have been returned
        assert_pfs_history: {source: 0, target: 1, request_count: 4, routes_count: [3, 2, 1, 2]}

        # The listed routes have been returned for requests from source node 0 to target node 1
        assert_pfs_history:
          source: 0
          target: 1
          expected_routes:
            - ['0x00[...]01', '0x00[...]04']
            - ['0x00[...]01', '0x00[...]02', '0x00[...]04']

    Expected response from PFS debug endpoint:
        {
            "request_count": <int>,
            "responses": [
                {
                    "source": <address>,
                    "target": <address>,
                    "routes": [
                    {
                        "path": [<address>, <address>, ...],
                        "estimated_fee": <estimated_fee>
                    }
                        ...,
                    ]
                },
                ...
            ]
        }
    """

    _name = "assert_pfs_history"
    _url_template = (
        "{pfs_url}/api/v1/_debug/routes/{token_network_address}/{source_address}{extra_params}"
    )

    @property
    def _url_params(self):
        pfs_url = self._runner.yaml.settings.services.pfs.url
        if not pfs_url:
            raise ScenarioError("PFS tasks require settings.services.pfs.url to be set.")

        source = self._config["source"]
        if isinstance(source, str) and len(source) == 42:
            source_address = source
        else:
            source_address = self._runner.get_node_address(source)

        extra_params = ""
        if "target" in self._config:
            target = self._config["target"]
            if isinstance(target, str) and len(target) == 42:
                target_address = target
            else:
                target_address = self._runner.get_node_address(target)
            extra_params = f"/{target_address}"

        params = dict(
            pfs_url=pfs_url,
            token_network_address=self._runner.token_network_address,
            source_address=source_address,
            extra_params=extra_params,
        )
        return params

    def _process_response(self, response_dict: dict):
        exp_request_count = self._config.get("request_count")
        if exp_request_count:
            actual_request_count = response_dict["request_count"]
            if actual_request_count != exp_request_count:
                raise ScenarioAssertionError(
                    f"Expected request_count {exp_request_count} but got {actual_request_count}"
                )

        actual_routes_counts = [len(response["routes"]) for response in response_dict["responses"]]
        exp_routes_counts = self._config.get("routes_count")
        if exp_routes_counts:
            if isinstance(exp_routes_counts, int):
                request_count = exp_request_count if exp_request_count else 1
                exp_routes_counts = [exp_routes_counts] * request_count
            elif isinstance(exp_routes_counts, (list, tuple)):
                if len(exp_routes_counts) != len(actual_routes_counts):
                    raise ScenarioAssertionError(
                        f"Expected {len(exp_routes_counts)} routes but got "
                        f"{len(actual_routes_counts)}"
                    )

            loop_iterator = enumerate(zip(exp_routes_counts, actual_routes_counts))
            for i, (exp_route_count, actual_route_count) in loop_iterator:
                if exp_route_count != actual_route_count:
                    raise ScenarioAssertionError(
                        f"Expected route count {exp_route_count} but got {actual_route_count} "
                        f"at index {i}"
                    )

        actual_routes = [
            route["path"]
            for response in response_dict["responses"]
            for route in response["routes"]
            if response["routes"]
        ]

        exp_routes = self._config.get("expected_routes")
        if exp_routes:
            if len(exp_routes) != len(actual_routes):
                raise ScenarioAssertionError(
                    f"Expected {len(exp_routes)} routes but got {len(actual_routes)}."
                )
            for i, (exp_route, actual_route) in enumerate(zip(exp_routes, actual_routes)):
                exp_route_addr = [self._runner.get_node_address(node) for node in exp_route]
                if exp_route_addr != actual_route:
                    raise ScenarioAssertionError(
                        f"Expected route {exp_route} but got {actual_route} at index {i}"
                    )

        exp_fees = self._config.get("expected_fees")
        if exp_fees:
            actual_fees = [
                route["estimated_fee"]
                for response in response_dict["responses"]
                for route in response["routes"]
                if response["routes"]
            ]
            if len(exp_fees) != len(actual_fees):
                raise ScenarioAssertionError(
                    f"Expected {len(exp_fees)} fees but got {len(actual_fees)}."
                )
            for i, (exp_fee, actual_fee) in enumerate(zip(exp_fees, actual_fees)):
                if exp_fee != actual_fee:
                    raise ScenarioAssertionError(
                        f"Expected fee {exp_fee} but got {actual_fee} at index {i}"
                    )

class AssertPFSIoUTask(RESTAPIActionTask):
    """
    Assert PFS IoU task

    Example usages:

        # IoUs of node 0 is stored at pfs with current amount 111
        assert_pfs_iou: {source: 0, amount: 111}

        # IoUs of node 0 is not stored in the pfs
        assert_pfs_iou: {source: 0, iou_exists: false}

    Expected response from PFS iou debug endpoint:
            {
                "sender": <address>,
                "amount": <int>,
                "exp_block": <int>,
            },

    """

    _name = "assert_pfs_iou"
    _url_template = "{pfs_url}/api/v1/_debug/ious/{source_address}"

    def __init__(
        self,
        runner: scenario_runner.ScenarioRunner,
        config: Any,
        parent: Task = None,
        abort_on_fail: bool = True,
    ) -> None:
        super().__init__(runner, config, parent, abort_on_fail)

        if "source" not in config:
            raise ScenarioError("Not all required keys provided. Required: source ")

        if not any(k in config for k in ["iou_exists", "amount"]):
            raise ScenarioError(f"Expected either iou_exists or amount.")

    @property
    def _url_params(self):
        pfs_url = self._runner.yaml.settings.services.pfs.url
        if not pfs_url:
            raise ScenarioError("PFS tasks require settings.services.pfs.url to be set.")

        source = self._config["source"]
        if isinstance(source, str) and len(source) == 42:
            source_address = source
        else:
            source_address = self._runner.get_node_address(source)

        params = dict(pfs_url=pfs_url, source_address=source_address)
        return params

    def _process_response(self, response_dict: dict):

        if self._config.get("iou_exists", True) is False:
            if response_dict:
                raise ScenarioAssertionError(f"Expected no IOU but got {response_dict}.")
        elif not response_dict:
            raise ScenarioAssertionError(f"Expected IOU, but no IOU exists.")
        else:
            exp_iou_amount = int(self._config["amount"])
            actual_iou_amount = int(response_dict["amount"])
            if actual_iou_amount != exp_iou_amount:
                raise ScenarioAssertionError(
                    f"Expected amount of {exp_iou_amount} but got {actual_iou_amount}."
                )


import subprocess
from .executables import sigenMorphExtractor, sigenSmoothing
import logging
from .auxFuncs import log_subprocess_output

def runSIGEN(inDir: str, outFile: str, VT: int, DT: int, C: int, S: int,
             XY_scale: float, Z_scale: float, threshold: int):

    toRun = [
              "mono", sigenMorphExtractor,
              "-i", inDir,
              "-o", outFile,
              "-t", "1",
              "-v", str(VT),
              "-d", str(DT),
              "-a", str(C),
              "-s", str(S),
              "-x", str(XY_scale),
              "-z", str(Z_scale),
              "-b", str(threshold)
            ]

    logging.info("[runSIGEN] Running {}".format(toRun))
    compProc = subprocess.run(toRun,
                   timeout=30 * 60,
                   stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    log_subprocess_output(compProc.stdout, "SIGENMorphExtractor")

def runSIGENSmoothing(inFile: str, outFile: str, smoothingLevel: int):

    toRun = [
        "mono", sigenSmoothing,
        inFile, outFile, str(smoothingLevel)
    ]

    logging.info("[runSIGENSmoothing] Running {}".format(toRun))
    compProc = subprocess.run(toRun,
                              timeout=30 * 60,
                              stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    log_subprocess_output(compProc.stdout, "SIGENSmoothing")


class Head:
    def __init__(self):
        self.x = 0
        self.y = 0
    def move(self, direction):
        if direction == "L":
            self.x -= 1
        elif direction == "R":
            self.x += 1
        elif direction == "D":
            self.y -= 1
        elif direction == "U":
            self.y += 1

class Tail:
    def __init__(self, head):
        self.x = 0
        self.y = 0
        self.visited = {(0, 0)}
        self.head = head
    def move(self):
        if self.head.x - self.x > 1:
            self.x += 1
            if self.head.y - self.y > 0:
                self.y += 1
            elif self.head.y - self.y < 0:
                self.y -= 1
        elif self.head.x - self.x < -1:
            self.x -= 1
            if self.head.y - self.y > 0:
                self.y += 1
            elif self.head.y - self.y < 0:
                self.y -= 1
        elif self.head.y - self.y > 1:
            self.y += 1
            if self.head.x - self.x > 0:
                self.x += 1
            elif self.head.x - self.x < 0:
                self.x -= 1
        elif self.head.y - self.y < -1:
            self.y -= 1
            if self.head.x - self.x > 0:
                self.x += 1
            elif self.head.x - self.x < 0:
                self.x -= 1
        self.visited.add((self.x, self.y))

def part1(filename):
    head = Head()
    tail = Tail(head)
    with open(filename, 'r') as f:
        lines = f.readlines()
    for line in lines:
        direction, magnitude = line.split()
        for _ in range(int(magnitude)):
            head.move(direction)
            tail.move()
    return len(tail.visited)

def part2(filename):
    knots = []
    knots.append(Head())
    for i in range(9):
        knots.append(Tail(head=knots[i]))
    with open(filename, 'r') as f:
        lines = f.readlines()
    for line in lines:
        direction, magnitude = line.split()
        for _ in range(int(magnitude)):
            knots[0].move(direction)
            for knot in knots[1:]:
                knot.move()
    return len(knots[9].visited)

print("Part 1:", part1("day9/data.txt"))
print("Part 2:", part2("day9/data.txt"))


import logging

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

formatter = logging.Formatter("%(asctime)s | %(name)s |  %(levelname)s: %(message)s")

file_handler = logging.FileHandler("log.log")
file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(formatter)
stream_handler = logging.StreamHandler()
stream_handler.setLevel(logging.INFO)

logger.addHandler(file_handler)
logger.addHandler(stream_handler)


import cv2
from time import sleep
cap = cv2.VideoCapture(0)
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
exposures = [-7, -5, -3, -1]
filenames = ["img_0.033.jpg", "img_0.25.jpg", "img_2.5.jpg", "img_15.jpg"]
cap.read()
for i in range(4):
    cap.set(cv2.CAP_PROP_EXPOSURE, exposures[i])
    sleep(1)
    ret, frame = cap.read()
    frame = cv2.flip(frame, 1)
    cv2.imshow(f'frame-{i}', frame)
    cv2.imwrite(filenames[i], frame)
cv2.waitKey(0)
cap.release()
cv2.destroyAllWindows()


# -*- coding: utf-8 -*-
"""
  Created on Thu Feb 20 21:32:44 2020
  一维问题的梯度下降算法
  @author: 刘瑜
  源代码文件是：U11GadientDescent1.py
"""
import numpy as np
import matplotlib.pyplot as plt
def f(x):  #目标函数，这里是f(x)=x**2+1
    return np.array(x)**2 + 1 #求f(x)

def d1(x): #对f(x)求导，即是目标函数的梯度
    return x * 2  #对f(x)求导

def gradient_descent_1d(cur_x=0.1, learn_rate=0.01, e=0.001, count=5000):
    #cur_x为初始x值，learn_rate为学习率，e为误差，count为最大迭代次数
    for i in range(count):
        grad=d1(cur_x)       #求当前x的梯度
        if abs(grad) <e:   #梯度收敛到指定误差内
            break                       #退出迭代过程
        cur_x=cur_x-grad*learn_rate #一维梯度下降迭代公式
        print("第%d次迭代逼近值为 %f"%(i+1,cur_x))
    print("最小值 ：", cur_x)
    print('最小值保留小数点后6位%.6f'%(cur_x))
    return cur_x
def ShowLine(minX,minY):    #显示目标函数曲线，及梯度下降最小值逼近情况
    x=[x for x in range(10)]+[x*(-1) for x in range(1,10)]
    x.sort()
    print(x)
    plt.plot(x,f(x))         #显示目标曲线
    plt.plot(minX,minY,'ro') #显示逼近的最小值
    plt.show()

minValue=gradient_descent_1d(cur_x=1, learn_rate=0.2, e=0.0001,count=10)
minY=f(minValue)
print('目标函数f(x)的最小值约是',minY)
ShowLine(minValue,minY)


import logging
from enum import Enum
from typing import Sequence, Set, Union, Type, TypeVar

from injectable.container.injection_container import InjectionContainer
from injectable.container.injectable import Injectable
from injectable.errors import InjectionError

T = TypeVar("T")

class RegistryType(Enum):
    CLASS = "class"
    QUALIFIER = "qualifier"

def get_dependency_registry_type(dependency: Union[Type[T], str]) -> RegistryType:
    return RegistryType.QUALIFIER if isinstance(dependency, str) else RegistryType.CLASS

def get_namespace_injectables(
    dependency_name: str, registry_type: RegistryType, namespace: str
) -> Set[Injectable]:
    if len(InjectionContainer.NAMESPACES) == 0:
        logging.warning(
            "Injection Container is empty. Make sure 'load_injection_container'"
            " is being called before any injections are made."
        )
    injection_namespace = InjectionContainer.NAMESPACES.get(namespace)
    if not injection_namespace:
        return set()
    registry = (
        injection_namespace.class_registry
        if registry_type is RegistryType.CLASS
        else injection_namespace.qualifier_registry
    )
    injectables = registry.get(dependency_name)
    return injectables

def filter_by_group(
    matches: Set[Injectable],
    group: str = None,
    exclude_groups: Sequence[str] = None,
) -> Set[Injectable]:
    exclude = exclude_groups or []
    matches = {
        inj
        for inj in matches
        if (group is None or inj.group == group) and inj.group not in exclude
    }
    return matches

def resolve_single_injectable(
    dependency_name: str, registry_type: RegistryType, matches: Set[Injectable]
) -> Injectable:
    if len(matches) == 1:
        for injectable in matches:
            return injectable

    primary_matches = [inj for inj in matches if inj.primary]
    if len(primary_matches) == 0:
        raise InjectionError(
            f"No primary injectable registered for {registry_type.value}:"
            f" '{dependency_name}'. Unable to resolve unambiguously: {len(matches)}"
            f" possible matches."
        )
    if len(primary_matches) > 1:
        raise InjectionError(
            f"Found {len(primary_matches)} injectables registered as primary for "
            f"{registry_type.value}: '{dependency_name}'. Unable to resolve"
            f" unambiguously."
        )
    return primary_matches[0]


import subprocess

from codeinterpreter.utils.error import PipInstallError
from codeinterpreter.utils.log import codeinterpreter_log

def pip_install(package: str, warn: bool = False):
    """
    Use subprocess execute `pip install`
    """
    cmd = f"pip install -q {package}"
    try:
        if warn and input(f"Install {package}? Y/n:") != "Y":
            raise ModuleNotFoundError(f"No module named {package}")
        print(f"start to install package: {package}")
        subprocess.check_call(cmd, shell=True)
        print(f"successfully installed package: {package}")
        codeinterpreter_log.info("%s installed successfully!", package)

    except subprocess.CalledProcessError as e:
        raise PipInstallError(package) from e


import requests, json

user_inputs = {}
Server = input("Enter the FQDN or IP of your Cb Protection Server : ")
API = input('Enter your API Token : ')
Hash = input('Enter the SHA256 of the file to ban : ')

#Prep request header and URL
authJson={
'X-Auth-Token': '{0}'.format(API), #Replace with actual API token
'content-type': 'application/json'
}
b9StrongCert = False #Set to True if the CbP server has a signed IIS cert
url = 'https://{0}/api/bit9platform/v1/fileRule'.format(Server) # Replace with server DNS/IP

#Contents of the API request being made
data = {'hash': '{0}'.format(Hash), 'fileState': 3}
r = requests.post(url, json.dumps(data), headers=authJson, verify=b9StrongCert)
r.raise_for_status() # Check if the call was successful
fileRule = r.json() # get resulting file rule object in JSON


#!/usr/local/bin/python3

import textwrap
#from modules import tiles
from modules.bgcolors import BgColors

class aMap(object):

	@classmethod
	def __act__(cls, item=None, **kwargs):
		"""Show the map of where the player has been."""
		player = kwargs['player']

		print(textwrap.fill("{}Printing map of where you have been.  Tiles are displayed with x,y coordinates for easy location reference.{}\n".format(BgColors.WARNING, BgColors.ENDC),70))

		# set grid to view.  Grid will present visited tiles/rooms within a x,y range of current location
		# set positive x,y grid ranges in tuple form
		axis_x = 10
		axis_y = 10

		normalized_grid_origin = (axis_x/2, axis_y/2)

		# create an empty list to hold the tiles where we have been
		buf = []
		tot = vis = enemy_tot = enemy_vis = 0
		# print the grid based on current coordinate view
		for key,tile in player.world.__dict__.items() :
			if isinstance(tile,kwargs['tiles'].MapTile) :
				tot += 1
				enemy_tot += 1 if isinstance(tile, kwargs['tiles'].EnemyRoom) else 0
				if tile.is_visited() :
					vis += 1
					enemy_vis += 1 if isinstance(tile, kwargs['tiles'].EnemyRoom) else 0

					# ensure the x,y axis position is within our range (set above) from center-tile (current location)
					if ( abs( tile.x - player.room.x ) <= normalized_grid_origin[0] ) and ( abs( tile.y - player.room.y ) <= normalized_grid_origin[1] ):
						# if the position logic is true, then add the tile to the buffer list for next iteration set
						buf.append( tile )

		buf = sorted(buf, key=lambda b:(b.y, b.x))

		print("\n{}World Stats: {}% Visited - {}/{} Areas. {}/{} Enemies. {}".format(BgColors.NORMAL, round(vis/tot*100,2), vis, tot, enemy_vis, enemy_tot, BgColors.ENDC))

		print("\n{}{} = Current Location".format(BgColors.WARNING, u"\u2588"*2))
		print("{}{} = 1st level".format(BgColors.SKYBLUE, u"\u2588"*2))
		print("{}{} = 2nd level".format(BgColors.CADETBLUE, u"\u2588"*2))

		line_text = ""
		# iterate over y axis grid range, each line will print a new position tile
		y_min = int(0 if player.room.y - normalized_grid_origin[1] < 0 else player.room.y - normalized_grid_origin[1])
		y_max =	int(axis_y if player.room.y + normalized_grid_origin[1] < axis_y else player.room.y + normalized_grid_origin[1])

		x_min = int(0 if player.room.x - normalized_grid_origin[0] < 0 else player.room.x - normalized_grid_origin[0])
		x_max = int(axis_x if player.room.x + normalized_grid_origin[0] < axis_x else player.room.x + normalized_grid_origin[0])

		for y in range(y_min,y_max) :
			# break to a new line and add the prefixing grid pipe
			#line_text += "\n{:<3}   | ".format(y)
			print("\n{}{:<4}| ".format(BgColors.NORMAL, y), end='' )
			# # iterate over y axis grid range, each line will print a new position tile
			#for x in range(0,axis_x) :
			for x in range(x_min,x_max) :
				found_x = False
				# iterate over the tiles in buf to get the x and y coordinate, to see if something exists on the x,y coordinate plane we are on currently
				for tile in buf :
					# if the tile x position matches the current iterated x range, and the y position matches the current iterated y range, then provide a square
					if tile.x == x and tile.y == y :
						found_x = True
						color_ = None
						if tile.floor == 1 :
							color_ = BgColors.SKYBLUE
						elif tile.floor == 2 :
							color_ = BgColors.CADETBLUE
						elif tile.floor == 3 :
							color_ = BgColors.WARNING
						else :
							color_ = BgColors.FAIL

						#line_text += "{} ".format(u"\u2588"*2) if tile.x == player.room.x and tile.y == player.room.y else "{} ".format(u"\u2591"*2)
						s = "{}{} ".format(BgColors.WARNING, u"\u2588"*2) if tile.x == player.room.x and tile.y == player.room.y else "{}{} ".format(color_, u"\u2588"*2)
						#s = "{}{} ".format(color_, "  ") if tile.x == player.room.x and tile.y == player.room.y else "{}{} ".format(BgColors.NORMAL, "  ")
						print(s, end='')

				# if we didn't find an x coordinate tile during our x axis loop, then provide empty spaces
				if not found_x :
					#line_text += "   "
					print("   ", end='' )

		#print( line_text )
		print('')

		# print bottom x axis grid
		print("      {}{}".format(BgColors.NORMAL, "―― "*axis_x)) # 2 underscores = 2 x axis spaces per tile
		# print the x coordinate integers
		x_axis_int = ''
		print( '    ',''.join([x_axis_int + " {}{} ".format(BgColors.NORMAL, x) for x in range(x_min,x_max)]) )


#get_ipython().run_line_magic('matplotlib', 'inline')
from fastai.basics import *

# In this part of the lecture we explain Stochastic Gradient Descent (SGD) which is an optimization method commonly used in neural networks. We will illustrate the concepts with concrete examples.

# ## Linear Regression problem
# The goal of linear regression is to fit a line to a set of points.

n=100

x = torch.ones(n,2)
x[:,0].uniform_(-1.,1)
x[:5]

#%%

a = tensor(3.,2); a

#%%
y = x@a + torch.rand(n)

#%% [markdown]
# In python, @ is matrix production

#%%
plt.scatter(x[:,0], y);

#%% [markdown]
# You want to find parameters (weights) a such that you minimize the error between the points and the line x@a. Note that here a is unknown. For a regression problem the most common error function or loss function is the mean squared error.

#%%


#!/usr/bin/env python3
import sys, os, struct, enum, time, re, math, collections, warnings

#######################
# Midi File Constants #
#######################

STATUS_MASK = 0xf0
CHANNEL_MASK = 0x0f

class HexInt(int):
    def __new__(cls, *args, **kwargs):
        return super().__new__(cls, *args, **kwargs)
    def __repr__(self):
        return hex(self)
    def __str__(self):
        return repr(self)

class MidiStatus(HexInt, enum.Enum):
    NoteOff    = 0x80
    NoteOn     = 0x90
    KeyPress   = 0xA0
    CtrlChange = 0xB0
    ProgChange = 0xC0
    ChannPress = 0xD0
    PitchBend  = 0xE0

    SysEx      = 0xF0
    SysExEsc   = 0xF7
    Meta       = 0xFF

    NonMidi    = 0xF0

NoteOff = MidiStatus.NoteOff
NoteOn = MidiStatus.NoteOn
KeyPress = MidiStatus.KeyPress
CtrlChange = MidiStatus.CtrlChange
ProgChange = MidiStatus.ProgChange
ChannPress = MidiStatus.ChannPress
PitchBend = MidiStatus.PitchBend
SysEx = MidiStatus.SysEx
SysExEsc = MidiStatus.SysExEsc
Meta = MidiStatus.Meta
NonMidi = MidiStatus.NonMidi

class MetaEvent(HexInt, enum.Enum):
    SeqNumber = 0x00
    TextEvent = 0x01
    Copyright = 0x02
    TrackName = 0x03
    InstrName = 0x04
    Lyric     = 0x05
    Marker    = 0x06
    CuePoint  = 0x07
    ProgName  = 0x08
    DevName   = 0x09
    ChannPref = 0x20
    MIDIPort  = 0x21
    EndOfTrk  = 0x2f
    SetTempo  = 0x51
    SMPTEOff  = 0x54
    TimeSig   = 0x58
    KeySig    = 0x59
    Specific  = 0x7f

N_DATA_BYTES = {
    NoteOff: 2, NoteOn: 2, KeyPress: 2, CtrlChange: 2, ProgChange: 1,
    ChannPress: 1, PitchBend: 2,
}

############################
# Midi File Classes/Parser #
############################

class MidiStatusByte(int):
    @property
    def status(self):
        return MidiStatus(self & STATUS_MASK)
    @property
    def channel(self):
        return self & CHANNEL_MASK
    def __repr__(self):
        return f'<{self.status.name}|{self.channel}: {self:#x}>'
    def __str__(self):
        return repr(self)

class RelMidiEvents(list):
    def __repr__(self):
        return f'RelMidiEvents({super().__repr__()})'

    def to_abs(self):
        events = AbsMidiEvents()
        tick = 0
        for evt, dt in self:
            tick += dt
            events.append((evt, tick))
        return events

class AbsMidiEvents(list):
    def __repr__(self):
        return f'AbsMidiEvents({super().__repr__()})'

    def to_rel(self):
        events = RelMidiEvents()
        lasttick = 0
        for evt, tick in self:
            events.append((evt, tick - lasttick))
            lasttick = tick
        return events

class ScheduledMidiEvents(list):
    def __repr__(self):
        return f'ScheduledMidiEvents({super().__repr__()})'

SmpteDivision = collections.namedtuple('SmpteDivision', ['fps', 'tpf'])

class MidiFile:
    def __init__(self, tracks, division, format=1):
        self.tracks = list(tracks)
        self.division = division
        self.format = format

    def __repr__(self):
        return (f'<MidiFile ntracks={len(self.tracks)} '
                f'division={self.division} format={self.format}>')

    def merged_events(self):
        events = AbsMidiEvents()
        for track in self.tracks:
            events.extend(track.to_abs())
        events.sort(key=lambda evt: evt[1])
        return events

    def schedule_events(self, sysex=False, meta=False):
        tempo = 500_000
        if isinstance(self.division, int):
            sec_per_tick = tempo / (self.division * 1000_000)
        else:
            sec_per_tick = 1 / (self.division.fps * self.division.tpf)
        midievents = ScheduledMidiEvents()
        last_tick = last_ts = 0
        for evt, tick in self.merged_events():
            ts = last_ts + (tick - last_tick) * sec_per_tick
            last_tick, last_ts = tick, ts
            if evt[0] == Meta and evt[1] == MetaEvent.SetTempo:
                tempo = evt[2]
                if isinstance(self.division, int):
                    sec_per_tick = tempo / (self.division * 1000_000)
            if (evt[0] in (SysEx, SysExEsc) and sysex or
                    evt[0] == Meta and meta or
                    evt[0] < NonMidi):
                midievents.append((evt, ts))
        return midievents

    @classmethod
    def read(cls, file):
        if isinstance(file, cls):
            return file
        elif isinstance(file, str):
            with open(file, 'rb') as file:
                buffer = file.read()
        else:
            buffer = file.read()
        chunk_head_fmt = struct.Struct('>4sI')
        header_data_fmt = struct.Struct('>HHh')
        tracks = []
        fmt = div = None
        i = 0
        while i < len(buffer):
            typ, length = chunk_head_fmt.unpack_from(buffer, i)
            i += chunk_head_fmt.size
            if typ == b'MThd':
                if fmt is not None:
                    raise ValueError('failed to parse midi file: multiple header chunks found')
                fmt, _ntrks, div = header_data_fmt.unpack_from(buffer, i)
                if div & 0x8000:
                    div = SmpteDivision(-(div >> 8), div & 0xff)
            elif fmt is None:
                raise ValueError('failed to parse midi file: no header chunk found')
            elif typ == b'MTrk':
                tracks.append(parse_track_data(buffer, i, length))
            else:
                warnings.warn(f'unknown chunk type: {typ}')
            i += length
        return cls(tracks, div, fmt)

def parse_track_data(buffer, offset=0, length=None):
    i = offset
    end = offset + length if length is not None else len(buffer)
    events = RelMidiEvents()
    running_status = 0
    while i < end:
        dt, i = parse_vlq(buffer, i)
        status = buffer[i]
        i += 1
        # Sysex Event
        if status in (SysEx, SysExEsc):
            length, i = parse_vlq(buffer, i)
            event = (MidiStatus(status), buffer[i: i+length])
            i += length
        # Meta Event
        elif status == Meta:
            try:
                typ = MetaEvent(buffer[i])
            except ValueError:
                typ = HexInt(buffer[i])
            length, i = parse_vlq(buffer, i+1)
            data = metaevent_data(typ, buffer[i: i+length])
            event = (MidiStatus(status), typ, data)
            i += length
        # MIDI Event
        else:
            # Running status
            if not status & 0x80:
                status = running_status
                i -= 1
            running_status = status
            length = N_DATA_BYTES[status & STATUS_MASK]
            event = (MidiStatusByte(status), *buffer[i: i+length])
            i += length
        events.append((event, dt))
    return events

def parse_vlq(data, offset=0):
    i = offset
    n = 0
    msb = 1
    while msb:
        b = data[i]
        n = n << 7 | b & 0x7f
        msb = b >> 7
        i += 1
    return n, i

def metaevent_data(typ, data):
    if typ in {MetaEvent.SeqNumber, MetaEvent.ChannPref,
               MetaEvent.MIDIPort, MetaEvent.SetTempo}:
        data = int.from_bytes(data, 'big')
    elif typ == MetaEvent.SMPTEOff:
        data = tuple(data)
    elif typ == MetaEvent.TimeSig:
        data = data[0], 2**data[1], data[2], data[3]
    elif typ == MetaEvent.KeySig:
        data = struct.unpack('bB', data)
    elif typ == MetaEvent.EndOfTrk:
        data = None
    return data

########################
# Midi Event Utilities #
########################

def shift(events, dt):
    return [(evt, ts+dt) for evt, ts in events]

def slice(events, start, end=None):
    if end is None:
        end = events[-1][1]
    return [(evt, ts-start) for evt, ts in events if start <= ts <= end]

def filter_events(events, type=None, channel=None):
    if type is not None and not isinstance(type, (list, tuple, set)):
        type = (type,)
    if channel is not None and not isinstance(channel, (list, tuple, set)):
        channel = (channel,)
    ret = []
    for evt, dt in events:
        if (type is None
             or evt[0] in type
             or (evt[0] < NonMidi and evt[0] & STATUS_MASK in type)
             or (evt[0] == Meta and evt[1] in type)):
            if (channel is None
                 or (evt[0] < NonMidi and evt[0] & CHANNEL_MASK in channel)):
                ret.append((evt, dt))
    return ret

def get_notes(file=None, events=None, off=False, ticks=False):
    import numpy as np
    if events is None:
        mf = MidiFile.read(file)
        events = mf.merged_events() if ticks else mf.schedule_events()
    noteon = np.array([(t, e[1]) for e, t in events
                       if e[0] & STATUS_MASK == NoteOn and e[2] > 0])
    if noteon.size == 0:
        return tuple(np.array([], int) for i in range(3 if off else 2))
    if not off:
        return noteon[:, 0], noteon[:, 1]
    noteoff = np.array([(t, e[1]) for e, t in events
                        if e[0] & STATUS_MASK == NoteOff or
                           e[0] & STATUS_MASK == NoteOn and e[2] == 0])
    indon = np.argsort(noteon[:, 1], kind='mergesort')
    indoff = np.argsort(noteoff[:, 1], kind='mergesort')
    noteoff[indon] = noteoff[indoff]
    return noteon[:, 0], noteoff[:, 0], noteon[:, 1]

def get_notes_mido(midofile=None, off=False, ticks=False):
    import numpy as np
    import mido
    msgs = mido.merge_tracks(midofile.tracks) if ticks else list(midofile)
    msgs = list(mido.midifiles.tracks._to_abstime(msgs))
    noteon = np.array([(m.time, m.note) for m in msgs
                       if m.type == 'note_on' and m.velocity > 0])
    if noteon.size == 0:
        return tuple(np.array([], int) for i in range(3 if off else 2))
    if not off:
        return noteon[:, 0], noteon[:, 1]
    noteoff = np.array([(m.time, m.note) for m in msgs
                        if m.type == 'note_off' or
                           m.type == 'note_on' and m.velocity == 0])
    indon = np.argsort(noteon[:, 1], kind='mergesort')
    indoff = np.argsort(noteoff[:, 1], kind='mergesort')
    noteoff[indon] = noteoff[indoff]
    return noteon[:, 0], noteoff[:, 0], noteon[:, 1]

#########################
# Midi Player Utilities #
#########################

NOTE_MAP = {
    'Cb': -1,  'C':  0,  'C#':  1,  'Db':  1,  'D':  2,  'D#':  3,
    'Eb':  3,  'E':  4,  'E#':  5,  'Fb':  4,  'F':  5,  'F#':  6,
    'Gb':  6,  'G':  7,  'G#':  8,  'Ab':  8,  'A':  9,  'A#': 10,
    'Bb': 10,  'B': 11,  'B#': 12,
}

NOTE_NAMES = ['C', 'C#', 'D', 'D#', 'E', 'F', 'F#', 'G', 'G#', 'A', 'A#', 'B']

def parse_note(note):
    if isinstance(note, int):
        return note
    match = re.fullmatch(r'([A-Ga-g][#b♯♭]?)(-?[0-9])?', note)
    if not match:
        raise ValueError(f'invalid note: {note}')
    note, num = match.groups()
    note = note[0].upper() + note[1:].replace('♯', '#').replace('♭', 'b')
    num = int(num) if num else 4
    return NOTE_MAP[note] + 12*(num + 1)

def note_name(note):
    return NOTE_NAMES[note % 12] + str(note // 12 - 1)

def note_frequency(note):
    note = parse_note(note)
    return 440.0 * 2**((note - 69) / 12)

def frequency_to_note(freq):
    return round(math.log2(freq / 440.0) * 12) + 69

def scale(start, end, intervals):
    start = parse_note(start)
    end = parse_note(end)
    if start > end:
        return scale(end, start, intervals)[::-1]
    i = 0
    l = []
    while start <= end:
        l.append(start)
        start += intervals[i]
        i = (i + 1) % len(intervals)
    return l

MAJOR_SCALE = [2, 2, 1, 2, 2, 2, 1]
NATURAL_MINOR_SCALE = [2, 1, 2, 2, 1, 2, 2]
HARMONIC_MINOR_SCALE = [2, 1, 2, 2, 1, 3, 1]

def major_scale(start, end):
    return scale(start, end, MAJOR_SCALE)

def natural_minor_scale(start, end):
    return scale(start, end, NATURAL_MINOR_SCALE)

def harmonic_minor_scale(start, end):
    return scale(start, end, HARMONIC_MINOR_SCALE)

def pitch_bend_bytes(p):
    x = min(max(int(8192*(p+1)), 0), 16383)
    return x & 127, x >> 7

def pitch_bend_value(lowb, highb):
    x = highb << 7 | lowb
    return (x / 8192) - 1

INSTRUMENT_NAMES = [
# Piano
'Acoustic Grand Piano', 'Bright Acoustic Piano', 'Electric Grand Piano',
'Honky-tonk Piano', 'Electric Piano 1', 'Electric Piano 2', 'Harpsichord',
'Clavinet',
# Chromatic Percussion
'Celesta', 'Glockenspiel', 'Music Box', 'Vibraphone', 'Marimba', 'Xylophone',
'Tubular Bells', 'Dulcimer',
# Organ
'Drawbar Organ', 'Percussive Organ', 'Rock Organ', 'Church Organ', 'Reed Organ',
'Accordion', 'Harmonica', 'Tango Accordion',
# Guitar
'Acoustic Guitar (nylon)', 'Acoustic Guitar (steel)', 'Electric Guitar (jazz)',
'Electric Guitar (clean)', 'Electric Guitar (muted)', 'Overdriven Guitar',
'Distortion Guitar', 'Guitar harmonics',
# Bass
'Acoustic Bass', 'Electric Bass (finger)', 'Electric Bass (pick)',
'Fretless Bass', 'Slap Bass 1', 'Slap Bass 2', 'Synth Bass 1', 'Synth Bass 2',
# Strings
'Violin', 'Viola', 'Cello', 'Contrabass', 'Tremolo Strings',
'Pizzicato Strings', 'Orchestral Harp', 'Timpani',
# Ensemble
'String Ensemble 1', 'String Ensemble 2', 'Synth Strings 1', 'Synth Strings 2',
'Choir Aahs', 'Voice Oohs', 'Synth Voice', 'Orchestra Hit',
# Brass
'Trumpet', 'Trombone', 'Tuba', 'Muted Trumpet', 'French Horn', 'Brass Section',
'Synth Brass 1', 'Synth Brass 2',
# Reed
'Soprano Sax', 'Alto Sax', 'Tenor Sax', 'Baritone Sax', 'Oboe', 'English Horn',
'Bassoon', 'Clarinet',
# Pipe
'Piccolo', 'Flute', 'Recorder', 'Pan Flute', 'Blown Bottle', 'Shakuhachi',
'Whistle', 'Ocarina',
# Synth Lead
'Lead 1 (square)', 'Lead 2 (sawtooth)', 'Lead 3 (calliope)', 'Lead 4 (chiff)',
'Lead 5 (charang)', 'Lead 6 (voice)', 'Lead 7 (fifths)', 'Lead 8 (bass + lead)',
# Synth Pad
'Pad 1 (new age)', 'Pad 2 (warm)', 'Pad 3 (polysynth)', 'Pad 4 (choir)',
'Pad 5 (bowed)', 'Pad 6 (metallic)', 'Pad 7 (halo)', 'Pad 8 (sweep)',
# Synth Effects
'FX 1 (rain)', 'FX 2 (soundtrack)', 'FX 3 (crystal)', 'FX 4 (atmosphere)',
'FX 5 (brightness)', 'FX 6 (goblins)', 'FX 7 (echoes)', 'FX 8 (sci-fi)',
# Ethnic
'Sitar', 'Banjo', 'Shamisen', 'Koto', 'Kalimba', 'Bag pipe', 'Fiddle', 'Shanai',
# Percussive
'Tinkle Bell', 'Agogo', 'Steel Drums', 'Woodblock', 'Taiko Drum', 'Melodic Tom',
'Synth Drum', 'Reverse Cymbal',
# Sound Effects
'Guitar Fret Noise', 'Breath Noise', 'Seashore', 'Bird Tweet', 'Telephone Ring',
'Helicopter', 'Applause', 'Gunshot']

INSTRUMENTS = {k.lower(): i for i, k in enumerate(INSTRUMENT_NAMES)}

PERCUSSION_CHANNEL = 9

###############
# Midi Player #
###############

class MidiPlayer:
    def __new__(cls, output=None, instrument=None):
        if cls is MidiPlayer:
            cls = get_player_class()
        return super().__new__(cls)

    def __init__(self, output=None, instrument=None):
        if instrument is not None:
            try:
                self.set_instrument(instrument)
            except:
                self.close()
                raise

    def send_message(self, message):
        raise NotImplementedError

    def send_sysex(self, message):
        self.send_message(message)

    def note_on(self, note, velocity=127, channel=0):
        self.send_message([NoteOn + channel, parse_note(note), velocity])

    def note_off(self, note, velocity=0, channel=0):
        self.send_message([NoteOff + channel, parse_note(note), velocity])

    def pitch_bend(self, bend, channel=0):
        self.send_message([PitchBend + channel, *pitch_bend_bytes(bend)])

    def all_notes_off(self, channel=None, fallback=True):
        channels = [channel] if channel is not None else range(16)
        for ch in channels:
            # Channel Mode 120: All Sound Off
            self.send_message([CtrlChange + ch, 120, 0])
            if fallback:
                for note in range(128):
                    self.note_off(note, channel=ch)

    def set_instrument(self, instrument, channel=0):
        if isinstance(instrument, str):
            instrument = INSTRUMENTS[instrument.lower()]
        self.send_message([ProgChange + channel, instrument])

    def wait(self, duration=1.0):
        if duration > 0:
            time.sleep(duration)

    def time(self):
        return time.perf_counter()

    def play_midi(self, file=None, events=None, volume=1, tempo_scale=1,
                  start=0, loop=False, sysex=False, print_progress=True):
        if events is None:
            events = MidiFile.read(file).schedule_events(sysex=True, meta=True)
        tottime = events[-1][1] / tempo_scale
        notes_on = set()
        try:
            do_loop = True
            while do_loop:
                last_ts = 0
                t0 = self.time() - start
                for evt, ts in events:
                    ts /= tempo_scale
                    if ts < start and evt[0] & STATUS_MASK in (NoteOn, NoteOff):
                        continue
                    # Print before waiting to hide delay
                    if print_progress and last_ts != ts and last_ts >= start:
                        print(f'\r{fmt_time(last_ts)}/{fmt_time(tottime)}', end='')
                    self.wait(ts + t0 - self.time())
                    if sysex and evt[0] == SysEx:
                        self.send_sysex(b'\xf0' + evt[1])
                    elif sysex and evt[0] == SysExEsc:
                        self.send_sysex(evt[1])
                    elif evt[0] < NonMidi:
                        if evt[0] & STATUS_MASK == NoteOn and evt[2]:
                            if volume != 1:
                                evt = (evt[0], evt[1], min(int(evt[2]*volume), 127))
                            notes_on.add((evt[0] & CHANNEL_MASK, evt[1]))
                        elif evt[0] & STATUS_MASK in (NoteOn, NoteOff):
                            notes_on.discard((evt[0] & CHANNEL_MASK, evt[1]))
                        self.send_message(bytes(evt))
                    last_ts = ts
                do_loop = loop
                start=0
            self.wait(.5)
        except KeyboardInterrupt:
            pass
        finally:
            for ch, note in notes_on:
                self.note_off(note, channel=ch)
        if print_progress:
            print(f'\r{fmt_time(last_ts)}/{fmt_time(tottime)}')

    def play_note(self, note, duration=1.0, velocity=127, channel=0,
                  instrument=None):
        if instrument is not None:
            self.set_instrument(instrument, channel)
        if isinstance(note, str):
            note = parse_note(note) if note.strip() else -1
        if note is None or note < 0:
            self.wait(duration)
            return
        try:
            self.note_on(note, velocity, channel)
            self.wait(duration)
        finally:
            self.note_off(note, channel=channel)

    def play_notes(self, notes, duration=0.5, delay=0.0, velocity=127,
                   channel=0, instrument=None):
        if instrument is not None:
            self.set_instrument(instrument, channel)
        for note in notes:
            dur, dely = duration, delay
            if isinstance(note, tuple):
                if len(note) == 1:
                    note, = note
                elif len(note) == 2:
                    note, dur = note
                else:
                    dely, note, dur = note
            self.wait(dely)
            self.play_note(note, dur, velocity, channel)

    def close(self):
        pass

    def __del__(self):
        self.close()

    def __enter__(self):
        return self

    def __exit__(self, *args):
        self.close()

#################
# Midi Backends #
#################

_backend = None

DEFAULT_BACKEND = 'rtmidi'

def set_backend(backend):
    global _backend
    if backend:
        assert backend in _PLAYER_CLASSES
    _backend = backend

def get_backend():
    global _backend
    if not _backend:
        _backend = os.environ.get('MIDI_BACKEND', DEFAULT_BACKEND)
        if _backend == 'rtmidi':
            try:
                import rtmidi
            except ImportError:
                warnings.warn("Couldn't load rtmidi backend. Falling back to pygame.")
                _backend = 'pygame'
        elif _backend != 'pygame':
            raise ValueError(f'Unknown backend: {_backend}')
    return _backend

def get_player_class():
    return _PLAYER_CLASSES[get_backend()]

class RtMidiPlayer(MidiPlayer):
    def __init__(self, output=None, instrument=None):
        import rtmidi
        if isinstance(output, rtmidi.MidiOut):
            self.output = output
        else:
            self.output = rtmidi.MidiOut()
            self.output.open_port(output or 0)
        super().__init__(output, instrument)

    @staticmethod
    def list_outputs():
        import rtmidi
        output = rtmidi.MidiOut()
        return output.get_ports()

    def send_message(self, message):
        self.output.send_message(message)

    def close(self):
        if hasattr(self, 'output'):
            self.output.close_port()
            del self.output

class PygameMidiPlayer(MidiPlayer):
    def __init__(self, output=None, instrument=None):
        global pygame
        import pygame.midi
        pygame.midi.init()
        if isinstance(output, pygame.midi.Output):
            self.output = output
        else:
            if output is None:
                output = pygame.midi.get_default_output_id()
            self.output = pygame.midi.Output(output)
        super().__init__(output, instrument)

    @staticmethod
    def list_outputs():
        import pygame.midi
        pygame.midi.init()
        infos = []
        for i in range(pygame.midi.get_count()):
            di = pygame.midi.get_device_info(i)
            if di[3]:
                infos.append(di[1].decode())
        return infos

    def send_message(self, message):
        self.output.write_short(*message)

    def send_sysex(self, message):
        self.output.write_sys_ex(0, message)

    def wait(self, duration=1.0):
        pygame.time.delay(int(duration * 1000))

    def time(self):
        return pygame.midi.time() / 1000

    def close(self):
        if hasattr(self, 'output'):
            try:
                self.output.close()
            except:
                pass
            del self.output

_PLAYER_CLASSES = {'rtmidi': RtMidiPlayer, 'pygame': PygameMidiPlayer}

##################
# Misc Utilities #
##################

def fmt_time(time):
    m, s = divmod(int(time), 60)
    return f'{m}:{s:02}'

def parse_time(string):
    if ':' in string:
        m, s = string.split(':')
        return int(m)*60 + float(s)
    return float(string)

def play_midi(file=None, events=None, volume=1, tempo_scale=1, start=0,
              loop=False, sysex=False, print_progress=True, output=None):
    with MidiPlayer(output) as player:
        player.play_midi(
            file=file, events=events, volume=volume, tempo_scale=tempo_scale,
            start=start, loop=loop, sysex=sysex, print_progress=print_progress)

def play_notes(notes, duration=0.5, delay=0.0, velocity=127, channel=0,
               instrument=None, output=None):
    with MidiPlayer(output) as player:
        player.play_notes(notes, duration, delay, velocity, channel, instrument)

def all_notes_off(output=None):
    with MidiPlayer(output) as player:
        player.all_notes_off(fallback=True)

def list_outputs():
    return get_player_class().list_outputs()

def main():
    import argparse
    p = argparse.ArgumentParser()
    p.add_argument('file', nargs='?')
    p.add_argument('-v', '--volume', type=float, default=1.0)
    p.add_argument('-t', '--tempo-scale', type=float, default=1.0)
    p.add_argument('-s', '--start', type=parse_time, default=0)
    p.add_argument('-p', '--progress', action='store_true', default=True, help='(default)')
    p.add_argument('-P', '--no-progress', dest='progress', action='store_false')
    p.add_argument('-S', '--sysex', action='store_true')
    p.add_argument('-L', '--loop', action='store_true')
    p.add_argument('-o', '--output', type=int)
    p.add_argument('-b', '--backend')
    p.add_argument('-T', '--length', action='store_true')
    p.add_argument('-A', '--all-notes-off', action='store_true')
    p.add_argument('-l', '--list-outputs', action='store_true')
    args = p.parse_args()
    set_backend(args.backend)
    if args.list_outputs:
        for o in list_outputs():
            print(o)
        return
    if args.all_notes_off:
        all_notes_off(args.output)
        return
    if not args.file:
        p.error('midi file is required')
    if args.length:
        events = MidiFile.read(args.file).schedule_events()
        print(fmt_time(events[-1][1]))
    else:
        play_midi(
            args.file, volume=args.volume, tempo_scale=args.tempo_scale,
            start=args.start, loop=args.loop, sysex=args.sysex,
            print_progress=args.progress, output=args.output)

if __name__ == '__main__':
    main()


from search.settings import CONTENT_SERVICES_TO_INDEX, GATEWAY_URL
from tests.assets import get_asset

def test_populate(client, requests_mock, mocker):
    service_to_index = CONTENT_SERVICES_TO_INDEX[0]

    xml = get_asset('scholarly-articles-items-response.xml')
    requests_mock.get(f'{GATEWAY_URL}/{service_to_index}/items', text=xml)

    xml = get_asset('scholarly-articles-article1-response.xml')
    requests_mock.get(f'{GATEWAY_URL}/{service_to_index}/items/article1/versions/latest', text=xml)

    es_mock = mocker.patch('search.populate.index_document')

    response = client.post('/populate')
    assert response.status_code == 200
    es_mock.assert_called_once()

    kwargs = {'index': 'articles',
              'doc_type': 'article',
              'id': 'article1',
              'body': {'id': 'article1',
                       'service': 'scholarly-articles'}
              }
    es_mock.assert_called_with(**kwargs)


from keras.models import model_from_json
import os
import glob
import numpy as np
from PIL import Image
import keyboard
import time
from mss import mss
import cv2
import requests
import asyncio
import functools

os.environ['KMP_DUPLICATE_LIB_OK']='True'

#读取配置文件
screen_config = []
f = open("screen_config.txt", "r")
for line in f:
	screen_config.append(line)
f.close()

mon = {'left': int(screen_config[0]), 'top': int(screen_config[1]), 'width': int(screen_config[2]), 'height': int(screen_config[3])}
sct = mss()

grab_Width = int(screen_config[2])
grab_Height = int(screen_config[3])
width = int(grab_Width/2)
height = int(grab_Height/2)

# Load Model
model = model_from_json(open("model.json", "r").read())
model.load_weights("car_weight.h5")
print(model.summary())

framerate_time = time.time()
counter = 0
i = 0
current_framerate = 0
delay = 0.04
count_time = 0
last_result = 1

responseCount = 0
last_targetX = 0

while True:

    time_start = time.time()
    img = sct.grab(mon)
    img2 = np.array(img)

    #退出机制
    if cv2.waitKey(1) & 0xFF == ord('q'):
        cv2.destroyAllWindows()
        break

    #压缩图像
    im = Image.frombytes('RGB', img.size, img.rgb)
    im = np.array(im.convert("L").resize((width, height)))
    im = im / 255

    time_grab = time.time() - time_start
    time_start = time.time()

    X = np.array([im])
    X = X.reshape(X.shape[0], width, height, 1)
    r = model.predict(X)
    result = np.argmax(r)

    time_predict = time.time() - time_start

    last_result = result

    counter+=1
    if (time.time() - framerate_time) > 1 :
        current_framerate = counter / (time.time() - framerate_time)
        counter = 0
        framerate_time = time.time()
        delay -= 0.0001
        if delay < 0:
            delay = 0

    targetX = int(result/100 *grab_Width)

    new_targetX = int(((targetX - last_targetX)/30) + last_targetX)

    #确认中心点
    centerPointX = int(grab_Width / 2)
    mid = int(centerPointX+(new_targetX - centerPointX) / 2*1.5)
    TargetHeight = 100
    TargetHeightMid = int(TargetHeight/2)

    #绘制参考图案
    cv2.circle(img2, (new_targetX, grab_Height-TargetHeight), 15, (0, 255, 0), 1,cv2.LINE_AA)

    cv2.line(img2, (centerPointX, grab_Height), (mid, grab_Height - TargetHeightMid), (255, 150, 0), 13,cv2.LINE_AA)
    cv2.line(img2, (mid, grab_Height-TargetHeightMid), (new_targetX, grab_Height - TargetHeight), (255, 150, 0), 10,cv2.LINE_AA)

    last_targetX = new_targetX

    #向夏力维发送坐标
    responseCount += 1
    if (responseCount > 30):    #发送间隔
        #response = requests.get("http://127.0.0.1:8088/api/xc/action_message?a=%s&b=%s&c=1&d=1"%(result, 0))
        responseCount = 0

    #CV渲染
    cv2.namedWindow('CV', 0)
    cv2.imshow('CV', img2)

    i += 1
    time.sleep(delay)


import os

import paddle.fluid as fluid
from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import fleet

from paddlerec.core.trainer import Trainer
from paddlerec.core.utils import envs
from paddlerec.core.utils import dataloader_instance
from paddlerec.core.reader import SlotReader

class TranspileTrainer(Trainer):
    def __init__(self, config=None):
        Trainer.__init__(self, config)
        device = envs.get_global_env("train.device", "cpu")
        if device == 'gpu':
            self._place = fluid.CUDAPlace(0)
            self._exe = fluid.Executor(self._place)
        self.processor_register()
        self.model = None
        self.inference_models = []
        self.increment_models = []

    def processor_register(self):
        print("Need implement by trainer, `self.regist_context_processor('uninit', self.instance)` must be the first")

    def _get_dataloader(self, state="TRAIN"):
        if state == "TRAIN":
            dataloader = self.model._data_loader
            namespace = "train.reader"
            class_name = "TrainReader"
        else:
            dataloader = self.model._infer_data_loader
            namespace = "evaluate.reader"
            class_name = "EvaluateReader"

        sparse_slots = envs.get_global_env("sparse_slots", None, namespace)
        dense_slots = envs.get_global_env("dense_slots", None, namespace)

        batch_size = envs.get_global_env("batch_size", None, namespace)
        print("batch_size: {}".format(batch_size))

        if sparse_slots is None and dense_slots is None:
            reader_class = envs.get_global_env("class", None, namespace)
            reader = dataloader_instance.dataloader(
                reader_class, state, self._config_yaml)
            reader_class = envs.lazy_instance_by_fliename(reader_class, class_name)
            reader_ins = reader_class(self._config_yaml)
        else:
            reader = dataloader_instance.slotdataloader("", state, self._config_yaml)
            reader_ins = SlotReader(self._config_yaml)

        if hasattr(reader_ins, 'generate_batch_from_trainfiles'):
            dataloader.set_sample_list_generator(reader)
        else:
            dataloader.set_sample_generator(reader, batch_size)

        debug_mode = envs.get_global_env("reader_debug_mode", False, namespace)
        if debug_mode:
            print("--- DataLoader Debug Mode Begin , show pre 10 data ---")
            for idx, line in enumerate(reader()):
                print(line)
                if idx >= 9:
                    break
            print("--- DataLoader Debug Mode End , show pre 10 data ---")
            exit(0)
        return dataloader

    def _get_dataset_ins(self):
        count = 0
        for f in self.files:
            for _, _ in enumerate(open(f, 'r')):
                count += 1
        return count

    def _get_dataset(self, state="TRAIN"):
        if state == "TRAIN":
            inputs = self.model.get_inputs()
            namespace = "train.reader"
            train_data_path = envs.get_global_env(
                "train_data_path", None, namespace)
        else:
            inputs = self.model.get_infer_inputs()
            namespace = "evaluate.reader"
            train_data_path = envs.get_global_env(
                "test_data_path", None, namespace)

        sparse_slots = envs.get_global_env("sparse_slots", None, namespace)
        dense_slots = envs.get_global_env("dense_slots", None, namespace)

        threads = int(envs.get_runtime_environ("train.trainer.threads"))
        batch_size = envs.get_global_env("batch_size", None, namespace)
        reader_class = envs.get_global_env("class", None, namespace)
        abs_dir = os.path.dirname(os.path.abspath(__file__))
        reader = os.path.join(abs_dir, '../utils', 'dataset_instance.py')

        if sparse_slots is None and dense_slots is None:
            pipe_cmd = "python {} {} {} {}".format(
                reader, reader_class, state, self._config_yaml)
        else:
            padding = envs.get_global_env("padding", 0, namespace)
            pipe_cmd = "python {} {} {} {} {} {} {} {}".format(
                reader, "slot", "slot", self._config_yaml, namespace, \
                sparse_slots.replace(" ", "#"), dense_slots.replace(" ", "#"), str(padding))

        if train_data_path.startswith("paddlerec::"):
            package_base = envs.get_runtime_environ("PACKAGE_BASE")
            assert package_base is not None
            train_data_path = os.path.join(
                package_base, train_data_path.split("::")[1])

        dataset = fluid.DatasetFactory().create_dataset()
        dataset.set_use_var(inputs)
        dataset.set_pipe_command(pipe_cmd)
        dataset.set_batch_size(batch_size)
        dataset.set_thread(threads)
        file_list = [
            os.path.join(train_data_path, x)
            for x in os.listdir(train_data_path)
        ]
        self.files = file_list
        dataset.set_filelist(self.files)

        debug_mode = envs.get_global_env("reader_debug_mode", False, namespace)
        if debug_mode:
            print(
                "--- Dataset Debug Mode Begin , show pre 10 data of {}---".format(file_list[0]))
            os.system("cat {} | {} | head -10".format(file_list[0], pipe_cmd))
            print(
                "--- Dataset Debug Mode End , show pre 10 data of {}---".format(file_list[0]))
            exit(0)

        return dataset

    def save(self, epoch_id, namespace, is_fleet=False):
        def need_save(epoch_id, epoch_interval, is_last=False):
            if is_last:
                return True

            if epoch_id == -1:
                return False

            return epoch_id % epoch_interval == 0

        def save_inference_model():
            save_interval = envs.get_global_env(
                "save.inference.epoch_interval", -1, namespace)

            if not need_save(epoch_id, save_interval, False):
                return

            feed_varnames = envs.get_global_env(
                "save.inference.feed_varnames", None, namespace)
            fetch_varnames = envs.get_global_env(
                "save.inference.fetch_varnames", None, namespace)
            if feed_varnames is None or fetch_varnames is None:
                return

            fetch_vars = [fluid.default_main_program().global_block().vars[varname]
                          for varname in fetch_varnames]
            dirname = envs.get_global_env(
                "save.inference.dirname", None, namespace)

            assert dirname is not None
            dirname = os.path.join(dirname, str(epoch_id))

            if is_fleet:
                fleet.save_inference_model(
                    self._exe, dirname, feed_varnames, fetch_vars)
            else:
                fluid.io.save_inference_model(
                    dirname, feed_varnames, fetch_vars, self._exe)
            self.inference_models.append((epoch_id, dirname))

        def save_persistables():
            save_interval = envs.get_global_env(
                "save.increment.epoch_interval", -1, namespace)

            if not need_save(epoch_id, save_interval, False):
                return

            dirname = envs.get_global_env(
                "save.increment.dirname", None, namespace)

            assert dirname is not None
            dirname = os.path.join(dirname, str(epoch_id))

            if is_fleet:
                fleet.save_persistables(self._exe, dirname)
            else:
                fluid.io.save_persistables(self._exe, dirname)
            self.increment_models.append((epoch_id, dirname))

        save_persistables()
        save_inference_model()

    def instance(self, context):
        models = envs.get_global_env("train.model.models")
        model_class = envs.lazy_instance_by_fliename(models, "Model")
        self.model = model_class(None)
        context['status'] = 'init_pass'

    def init(self, context):
        print("Need to be implement")
        context['is_exit'] = True

    def dataloader_train(self, context):
        print("Need to be implement")
        context['is_exit'] = True

    def dataset_train(self, context):
        print("Need to be implement")
        context['is_exit'] = True

    def infer(self, context):
        infer_program = fluid.Program()
        startup_program = fluid.Program()
        with fluid.unique_name.guard():
            with fluid.program_guard(infer_program, startup_program):
                self.model.infer_net()

        if self.model._infer_data_loader is None:
            context['status'] = 'terminal_pass'
            return

        reader = self._get_dataloader("Evaluate")

        metrics_varnames = []
        metrics_format = []

        metrics_format.append("{}: {{}}".format("epoch"))
        metrics_format.append("{}: {{}}".format("batch"))

        for name, var in self.model.get_infer_results().items():
            metrics_varnames.append(var.name)
            metrics_format.append("{}: {{}}".format(name))

        metrics_format = ", ".join(metrics_format)
        self._exe.run(startup_program)

        model_list = self.increment_models

        evaluate_only = envs.get_global_env(
            'evaluate_only', False, namespace='evaluate')
        if evaluate_only:
            model_list = [(0, envs.get_global_env(
                'evaluate_model_path', "", namespace='evaluate'))]

        is_return_numpy = envs.get_global_env(
            'is_return_numpy', True, namespace='evaluate')

        for (epoch, model_dir) in model_list:
            print("Begin to infer No.{} model, model_dir: {}".format(
                epoch, model_dir))
            program = infer_program.clone()
            fluid.io.load_persistables(self._exe, model_dir, program)
            reader.start()
            batch_id = 0
            try:
                while True:
                    metrics_rets = self._exe.run(
                        program=program,
                        fetch_list=metrics_varnames,
                        return_numpy=is_return_numpy)

                    metrics = [epoch, batch_id]
                    metrics.extend(metrics_rets)

                    if batch_id % 2 == 0 and batch_id != 0:
                        print(metrics_format.format(*metrics))
                    batch_id += 1
            except fluid.core.EOFException:
                reader.reset()

        context['status'] = 'terminal_pass'

    def terminal(self, context):
        print("clean up and exit")
        context['is_exit'] = True


if __name__ == '__main__':
    from check_item import check_item
else:
    from .check_item import check_item

def find_position(item_shift_code, item_line_length: int, check_order, pallet_shift_code, pallet_line_length: int):
    """Ищет возможное расположение объекта по принципу жадного алгоритма "как можно ниже, как можно левее"

    Returns:
        is_placed_item: bool
        x, y: int - растровые координаты возможного расположения"""
    x = 0
    y = 0
    shift = 0
    is_placed_item = False
    bad_line = 0
    while (not is_placed_item) and (y + item_shift_code.shape[0] <= pallet_shift_code.shape[0]):
        while (not is_placed_item) and (x + shift + item_line_length <= pallet_line_length):
            x += shift
            is_placed_item, shift, bad_line = check_item(item_shift_code, item_line_length, check_order, pallet_shift_code, x, y, bad_line)
        if not is_placed_item:
            y += 1
            x = 0
            shift = 0
    return is_placed_item, x, y


from aquilon.exceptions_ import ArgumentError
from aquilon.aqdb.model import RouterAddress, Building, ARecord
from aquilon.aqdb.model.network_environment import get_net_dns_env
from aquilon.aqdb.model.network import get_net_id_from_ip
from aquilon.worker.broker import BrokerCommand
from aquilon.worker.dbwrappers.change_management import ChangeManagement

class CommandUpdateRouterAddress(BrokerCommand):
    requires_plenaries = True

    def render(self, session, plenaries, fqdn, ip, building, clear_location,
               network_environment, dns_environment, comments,
               user, justification, reason, logger, **arguments):
        dbnet_env, dbdns_env = get_net_dns_env(session, network_environment,
                                               dns_environment)
        if fqdn:
            dbdns_rec = ARecord.get_unique(session, fqdn=fqdn,
                                           dns_environment=dbdns_env,
                                           compel=True)
            ip = dbdns_rec.ip
        if not ip:
            raise ArgumentError("Please specify either --ip or --fqdn.")

        dbnetwork = get_net_id_from_ip(session, ip, dbnet_env)

        # Validate ChangeManagement
        cm = ChangeManagement(session, user, justification, reason, logger, self.command, **arguments)
        cm.consider(dbnetwork)
        cm.validate()

        router = RouterAddress.get_unique(session, ip=ip, network=dbnetwork,
                                          compel=True)

        if building:
            dbbuilding = Building.get_unique(session, name=building,
                                             compel=True)
            router.location = dbbuilding
        elif clear_location:
            router.location = None

        if comments is not None:
            router.comments = comments

        # Refresh the plenaries even if nothing in the DB has changed
        plenaries.add(dbnetwork)
        plenaries.write()

        session.flush()