import sys
from optparse import OptionParser

from typing import Optional, Tuple
import time

import locale
locale.setlocale( locale.LC_ALL, '' )

from loguru import logger
logger.add("logs/ftx_static_cash_and_carry_perpetual.log", rotation="100 MB")

from cryptomancer.security_master import SecurityMaster

from cryptomancer.account.ftx_account import FtxAccount
from cryptomancer.exchange_feed.ftx_exchange_feed import FtxExchangeFeed

from cryptomancer.execution_handler.execution_session import execution_scope

from cryptomancer.execution_handler.limit_order import LimitOrder

def static_cash_and_carry(account: FtxAccount, exchange_feed: FtxExchangeFeed, underlying: str,
                        cash_collateral_target: float, cash_collateral_bounds: Tuple[float, float],
                        minimum_size: Optional[float] = 0.001,
                        force: Optional[bool] = False):
    underlying_name = f'{underlying}/USD'
    future_name = f'{underlying}-0924'

    # set up exchange subscriptions
    _ = exchange_feed.get_ticker(underlying_name)
    _ = exchange_feed.get_ticker(future_name)

    # get the current positions
    positions = account.get_positions()

    for position in positions:
        logger.info(f'Current Position | {position.name} | {position.side} | {position.size} | {locale.currency(position.usd_value, grouping = True)}')

    usd_coins = list(filter(lambda position: position.name == 'USD', positions))
    underlying_positions = list(filter(lambda position: position.name.upper() == underlying, positions))
    perpetual_positions = list(filter(lambda position: position.name.upper() == future_name, positions))

    # figure out how much cash collateral we have
    # we have to be careful here because USD will already *include* PERP gains/losses,
    # 	so we don't want to double-count those values
    portfolio_value = sum([position.usd_value for position in usd_coins]) + \
                        sum([position.usd_value for position in underlying_positions])

    logger.info(f'Current Portfolio Value | {locale.currency(portfolio_value, grouping = True)}')

    usd_value = sum([position.usd_value for position in usd_coins])
    underlying_size = sum([position.net_size for position in underlying_positions])
    perpetual_size = sum([position.net_size for position in perpetual_positions])

    # SHOULD THE LOGIC HERE ACCOUNT FOR TARGET POSITION VS CURRENT POSITION
    # e.g. WHAT DO WE DO WITH OTHER COINS / PERPS IN THE ACCOUNT?!  LIQUIDATE?

    margin_pct = usd_value / portfolio_value

    logger.info(f'Current Margin | ' + '{:.2%}'.format(margin_pct))

    # if we're within our collateral bounds, do nothing
    if force or (margin_pct < cash_collateral_bounds[0] or margin_pct > cash_collateral_bounds[1]):
        if (margin_pct < cash_collateral_bounds[0] or margin_pct > cash_collateral_bounds[1]):
            logger.info('Margin Target Out of Bounds (' + '{:.2%}'.format(cash_collateral_bounds[0]) + ', ' +
                                                    '{:.2%}'.format(cash_collateral_bounds[1]) + ')')
        else:
            logger.info('Trade Forced.')

        # otherwise, figure out what our target positions are
        target_margin_usd = portfolio_value * cash_collateral_target
        target_exposure_usd = portfolio_value * (1 - cash_collateral_target)

        current_exposure_usd = sum([position.usd_value for position in underlying_positions])

        target_usd_trade = target_exposure_usd - current_exposure_usd

        # go half way
        target_usd_trade = target_usd_trade / 2

        # we do the trade in the underlying first because we're trying to hit a specific
        # dollar amount; once we get that dollar amount executed, we can match it with
        # a corresponding trade in the perpetuals
        side = 'buy' if target_usd_trade > 1e-8 else 'sell'
        logger.info(f'{side.upper()} {locale.currency(target_usd_trade, grouping = True)} {underlying_name}')

        while True:
            try:
                market = exchange_feed.get_ticker(underlying_name)
                mid_point = (market['bid'] + market['ask']) / 2.
                width = (market['ask'] - market['bid']) / mid_point
            except:
                continue

            price = mid_point * (1 - width / 2) if side == 'buy' else mid_point * (1 + width / 2)
            size = abs(target_usd_trade) / price

            try:
                with execution_scope(wait = True, timeout = 10) as session:
                    underlying_order = LimitOrder(account = account,
                                                    market = underlying_name,
                                                    side = side,
                                                    size = size,
                                                    price = price,
                                                    post_only = True)

                    session.add(underlying_order)

            except TimeoutError:
                logger.debug(f'Timed out.  Retrying.')
                pass

            order_status = session.get_order_statuses()
            if len(order_status) == 0:
                # the order errored out
                logger.info(f'{side.upper()} {locale.currency(target_usd_trade, grouping = True)} {underlying_name} FAILED')
                continue

            else:
                order_status = order_status[0]

                while order_status.status == "open":
                    # make sure we don't get stuck in the loop; e.g. what happens if the cancel never got sent?
                    try:
                        ftx_account.cancel_order(order_status.order_id)
                    except:
                        pass
                    time.sleep(0.1)
                    order_status = session.get_order_statuses()[0]

                # figure out how much of the order was actually filled
                filled_size = order_status.filled_size if order_status.side == "buy" else -order_status.filled_size
                filled_price = order_status.average_fill_price

                if abs(filled_size) < 1e-8:
                    continue

                underlying_size = underlying_size + filled_size
                logger.info(f'Filled {filled_size} in {underlying_name} | Total: {underlying_size}')

                target_usd_trade = target_usd_trade - filled_size * filled_price

                # We might've gotten a partial fill; see if the remaining
                # capital to trade is less than the minimum trade size
                if abs(target_usd_trade) / filled_price < minimum_size:
                    break

    # if we have 5 underlying, we need -5 perpetuals
    # so we want to take that target and subtract what we already own
    perpetual_to_buy = (-underlying_size - perpetual_size)
    perpetual_to_buy = int(perpetual_to_buy / minimum_size) * minimum_size

    side = 'buy' if perpetual_to_buy > 1e-8 else 'sell'
    size = abs(perpetual_to_buy)

    if abs(perpetual_to_buy) > minimum_size:
        while True:
            market = exchange_feed.get_ticker(future_name)
            mid_point = (market['bid'] + market['ask']) / 2.
            width = (market['ask'] - market['bid']) / mid_point

            price = mid_point * (1 - width / 2) if side == 'buy' else mid_point * (1 + width / 2)

            try:
                with execution_scope(wait = True, timeout = 10) as session:
                    logger.info(f"{side.upper()} {size} {future_name}")
                    perpetual_order = LimitOrder(account = account,
                                        market = future_name,
                                        side = side,
                                        size = size,
                                        price = price,
                                        post_only = True)

                    session.add(perpetual_order)

            except TimeoutError:
                logger.debug(f'Timed out.  Retrying.')
                pass

            order_status = session.get_order_statuses()
            if len(order_status) == 0:
                # the order errored out
                logger.info(f'{side.upper()} {size} {future_name} FAILED')
                continue

            else:
                order_status = order_status[0]

                while order_status.status == "open":
                    time.sleep(0.1)
                    order_status = session.get_order_statuses()[0]

                filled_size = order_status.filled_size if order_status.side == "buy" else -order_status.filled_size

                if abs(filled_size) < 1e-8:
                    continue

                total_perpetual = sum([position.net_size for position in perpetual_positions])
                total_perpetual = total_perpetual + filled_size

                logger.info(f'Filled {filled_size} in {future_name} | Total: {total_perpetual}')
                break

if __name__ == '__main__':
    usage = "usage: " + sys.argv[0] + " <FTX Account Name> <Underlying> [optional-args]"
    parser = OptionParser(usage = usage)

    parser.add_option("-m", "--margin",
                      help="Margin target", type=float, dest="margin", default=0.2)
    parser.add_option("-l", "--margin-lower",
                      help="Lower bound threshold for margin (forced rebalance)", type=float, dest="margin_low", default=0.15)
    parser.add_option("-u", "--margin-upper",
                      help="Upper bound threshold for margin (forced rebalance)", type=float, dest="margin_high", default=0.25)
    parser.add_option("-f", "--force",
                      help="Force the trade to go through", dest="force", default=False, action="store_true")

    (options, args) = parser.parse_args()
    if len(args) < 2:
        print(usage)
        sys.exit(1)

    if options.margin < options.margin_low:
        print("Margin target must be higher than lower margin threshold.")
        sys.exit(0)

    if options.margin > options.margin_high:
        print("Margin target must be lower than upper margin threshold.")
        sys.exit(0)

    if options.margin < 0 or options.margin > 1:
        print("Margin target must be between 0-100%.")
        sys.exit(0)

    if options.margin_low < 0:
        print("Margin lower bound must be greater than 0%.")
        sys.exit(0)

    if options.margin_high > 1:
        print("Margin upper bound must be less than 100%.")
        sys.exit(0)

    account_name = args[0]
    underlying = args[1].upper()

    sm = SecurityMaster("FTX")
    try:
        market_spec = sm.get_market_spec(underlying + "/USD")
    except:
        logger.exception("Invalid underlying (does not exist in Securities Master database)")
        sys.exit(0)

    try:
        contract_spec = sm.get_contract_spec(underlying + '-PERP')
    except:
        logger.exception("No associated perpetual contract at FTX (does not exist in Securities Master database)")
        sys.exit(0)

    min_size = max(market_spec['sizeIncrement'], contract_spec['sizeIncrement'])

    try:
        ftx_account = FtxAccount(account_name)
        ftx_feed = FtxExchangeFeed(account_name)
    except Exception as e:
        logger.exception(e)
        sys.exit(0)

    static_cash_and_carry(ftx_account, ftx_feed, underlying,
                            cash_collateral_target = options.margin,
                            cash_collateral_bounds = (options.margin_low, options.margin_high),
                            minimum_size = min_size,
                            force = options.force)


import torch
import torch.nn as nn
import torchvision.models as models
from model.ResNet import B2_ResNet
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
from torch.nn import Parameter, Softmax
import torch.nn.functional as F
from model.HolisticAttention import HA
from torch.autograd import Variable
from torch.distributions import Normal, Independent, kl
import numpy as np
from model.Res2Net import res2net50_v1b_26w_4s

class Descriptor(nn.Module):
    def __init__(self, channel):
        super(Descriptor, self).__init__()
        self.relu = nn.ReLU(inplace=True)
        self.sconv1 = nn.Conv2d(3, channel, kernel_size=3, stride=2, padding=1)
        self.sconv2 = nn.Conv2d(64, 256, kernel_size=3, stride=1, padding=1)
        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
        self.sconv3 = nn.Conv2d(256, 512, kernel_size=3, stride=1, padding=1)
        self.sconv4 = nn.Conv2d(512, 1024, kernel_size=3, stride=1, padding=1)

        # self.bn1 = nn.BatchNorm2d(64)
        # self.bn2 = nn.BatchNorm2d(256)
        # self.bn3 = nn.BatchNorm2d(512)
        # self.bn4 = nn.BatchNorm2d(1024)
        self.layer5 = self._make_pred_layer(Classifier_Module, [6, 12, 18, 24], [6, 12, 18, 24], channel, 1024)
        self.conv_pred = nn.Conv2d(1, channel, 3, 1,1)
        self.conv1 = nn.Conv2d(channel*2, channel, 3, 2, 1)
        self.conv2 = nn.Conv2d(channel, channel, 3, 1, 1)
        self.conv3 = nn.Conv2d(channel, channel , 3, 2, 1)
        self.conv4 = nn.Conv2d(channel, channel, 3, 1, 1)
        self.conv5 = nn.Conv2d(channel, 1, 3, 2, 1)
        self.bn1 = nn.BatchNorm2d(channel)
        self.bn2 = nn.BatchNorm2d(channel)
        self.bn3 = nn.BatchNorm2d(channel)
        self.bn4 = nn.BatchNorm2d(channel)
        self.upsample = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)

        self.leaky_relu = nn.LeakyReLU(negative_slope=0.2, inplace=True)

    def _make_pred_layer(self, block, dilation_series, padding_series, NoLabels, input_channel):
        return block(dilation_series, padding_series, NoLabels, input_channel)

    def forward(self, input, seg):
        x1 = self.sconv1(input)
        # # x1 = self.bn1(x1)
        # x1 = self.relu(x1)
        # x2 = self.sconv2(x1)
        # # x2 = self.bn2(x2)
        # x2 = self.relu(x2)
        # x2 = self.maxpool(x2)
        # x3 = self.sconv3(x2)
        # # x3 = self.bn3(x3)
        # x3 = self.relu(x3)
        # x3 = self.maxpool(x3)
        # x4 = self.sconv4(x3)
        # # x4 = self.bn4(x4)
        # x4 = self.relu(x4)
        # x5 = self.layer5(x4)
        int_feat = self.upsample(x1)
        seg_conv = self.conv_pred(seg)
        feature_map = torch.cat((int_feat,seg_conv),1)
        x = self.conv1(feature_map)
        x = self.bn1(x)
        x = self.leaky_relu(x)
        x = self.conv2(x)
        x = self.bn2(x)
        x = self.leaky_relu(x)
        x = self.conv3(x)
        x = self.bn3(x)
        x = self.leaky_relu(x)
        x = self.conv4(x)
        x = self.bn4(x)
        x = self.leaky_relu(x)
        x = self.conv5(x)
        return x

class Encoder_x(nn.Module):
    def __init__(self, input_channels, channels, latent_size):
        super(Encoder_x, self).__init__()
        self.contracting_path = nn.ModuleList()
        self.input_channels = input_channels
        self.relu = nn.ReLU(inplace=True)
        self.layer1 = nn.Conv2d(input_channels, channels, kernel_size=4, stride=2, padding=1)
        self.bn1 = nn.BatchNorm2d(channels)
        self.layer2 = nn.Conv2d(channels, 2*channels, kernel_size=4, stride=2, padding=1)
        self.bn2 = nn.BatchNorm2d(channels * 2)
        self.layer3 = nn.Conv2d(2*channels, 4*channels, kernel_size=4, stride=2, padding=1)
        self.bn3 = nn.BatchNorm2d(channels * 4)
        self.layer4 = nn.Conv2d(4*channels, 8*channels, kernel_size=4, stride=2, padding=1)
        self.bn4 = nn.BatchNorm2d(channels * 8)
        self.layer5 = nn.Conv2d(8*channels, 8*channels, kernel_size=4, stride=2, padding=1)
        self.bn5 = nn.BatchNorm2d(channels * 8)
        self.channel = channels

        self.fc1_1 = nn.Linear(channels * 8 * 8 * 8, latent_size)
        self.fc2_1 = nn.Linear(channels * 8 * 8 * 8, latent_size)

        self.fc1_2 = nn.Linear(channels * 8 * 11 * 11, latent_size)
        self.fc2_2 = nn.Linear(channels * 8 * 11 * 11, latent_size)

        self.fc1_3 = nn.Linear(channels * 8 * 14 * 14, latent_size)
        self.fc2_3 = nn.Linear(channels * 8 * 14 * 14, latent_size)

        self.leakyrelu = nn.LeakyReLU()

    def forward(self, input):
        output = self.leakyrelu(self.bn1(self.layer1(input)))
        # print(output.size())
        output = self.leakyrelu(self.bn2(self.layer2(output)))
        # print(output.size())
        output = self.leakyrelu(self.bn3(self.layer3(output)))
        # print(output.size())
        output = self.leakyrelu(self.bn4(self.layer4(output)))
        # print(output.size())
        output = self.leakyrelu(self.bn4(self.layer5(output)))

        # print(output.size())
        if input.shape[2] == 256:
            # print('************************256********************')
            # print(input.size())
            output = output.view(-1, self.channel * 8 * 8 * 8)

            mu = self.fc1_1(output)
            logvar = self.fc2_1(output)
            dist = Independent(Normal(loc=mu, scale=torch.exp(logvar)), 1)
            # print(output.size())
            # output = self.tanh(output)

            return dist, mu, logvar
        elif input.shape[2] == 352:
            # print('************************352********************')
            #print(input.size())
            output = output.view(-1, self.channel * 8 * 11 * 11)

            mu = self.fc1_2(output)
            logvar = self.fc2_2(output)
            dist = Independent(Normal(loc=mu, scale=torch.exp(logvar)), 1)
            # print(output.size())
            # output = self.tanh(output)

            return dist, mu, logvar
        else:
            # print('************************bigger********************')
            # print(input.size())
            output = output.view(-1, self.channel * 8 * 14 * 14)

            mu = self.fc1_3(output)
            logvar = self.fc2_3(output)
            dist = Independent(Normal(loc=mu, scale=torch.exp(logvar)), 1)
            # print(output.size())
            # output = self.tanh(output)

            return dist, mu, logvar
        # output = output.view(-1, self.channel * 8 * 11 * 11)
        # # print(output.size())
        # # output = self.tanh(output)
        #
        # mu = self.fc1(output)
        # logvar = self.fc2(output)
        # dist = Independent(Normal(loc=mu, scale=torch.exp(logvar)), 1)
        # # print(output.size())
        # # output = self.tanh(output)
        #
        # return dist, mu, logvar

class Encoder_xy(nn.Module):
    def __init__(self, input_channels, channels, latent_size):
        super(Encoder_xy, self).__init__()
        self.contracting_path = nn.ModuleList()
        self.input_channels = input_channels
        self.relu = nn.ReLU(inplace=True)
        self.layer1 = nn.Conv2d(input_channels, channels, kernel_size=4, stride=2, padding=1)
        self.bn1 = nn.BatchNorm2d(channels)
        self.layer2 = nn.Conv2d(channels, 2*channels, kernel_size=4, stride=2, padding=1)
        self.bn2 = nn.BatchNorm2d(channels * 2)
        self.layer3 = nn.Conv2d(2*channels, 4*channels, kernel_size=4, stride=2, padding=1)
        self.bn3 = nn.BatchNorm2d(channels * 4)
        self.layer4 = nn.Conv2d(4*channels, 8*channels, kernel_size=4, stride=2, padding=1)
        self.bn4 = nn.BatchNorm2d(channels * 8)
        self.layer5 = nn.Conv2d(8*channels, 8*channels, kernel_size=4, stride=2, padding=1)
        self.bn5 = nn.BatchNorm2d(channels * 8)
        self.channel = channels

        self.fc1_1 = nn.Linear(channels * 8 * 8 * 8, latent_size)
        self.fc2_1 = nn.Linear(channels * 8 * 8 * 8, latent_size)

        self.fc1_2 = nn.Linear(channels * 8 * 11 * 11, latent_size)
        self.fc2_2 = nn.Linear(channels * 8 * 11 * 11, latent_size)

        self.fc1_3 = nn.Linear(channels * 8 * 14 * 14, latent_size)
        self.fc2_3 = nn.Linear(channels * 8 * 14 * 14, latent_size)

        self.leakyrelu = nn.LeakyReLU()

    def forward(self, x):
        output = self.leakyrelu(self.bn1(self.layer1(x)))
        # print(output.size())
        output = self.leakyrelu(self.bn2(self.layer2(output)))
        # print(output.size())
        output = self.leakyrelu(self.bn3(self.layer3(output)))
        # print(output.size())
        output = self.leakyrelu(self.bn4(self.layer4(output)))
        # print(output.size())
        output = self.leakyrelu(self.bn4(self.layer5(output)))
        # print(output.size())
        if x.shape[2] == 256:
            # print('************************256********************')
            # print(x.size())
            output = output.view(-1, self.channel * 8 * 8 * 8)

            mu = self.fc1_1(output)
            logvar = self.fc2_1(output)
            dist = Independent(Normal(loc=mu, scale=torch.exp(logvar)), 1)
            # print(output.size())
            # output = self.tanh(output)

            return dist, mu, logvar
        elif x.shape[2] == 352:
            # print('************************352********************')
            # print(x.size())
            output = output.view(-1, self.channel * 8 * 11 * 11)

            mu = self.fc1_2(output)
            logvar = self.fc2_2(output)
            dist = Independent(Normal(loc=mu, scale=torch.exp(logvar)), 1)
            # print(output.size())
            # output = self.tanh(output)

            return dist, mu, logvar
        else:
            # print('************************bigger********************')
            # print(x.size())
            output = output.view(-1, self.channel * 8 * 14 * 14)

            mu = self.fc1_3(output)
            logvar = self.fc2_3(output)
            dist = Independent(Normal(loc=mu, scale=torch.exp(logvar)), 1)
            # print(output.size())
            # output = self.tanh(output)

            return dist, mu, logvar

class Generator(nn.Module):
    def __init__(self, channel, latent_dim):
        super(Generator, self).__init__()
        self.sal_encoder = Saliency_feat_encoder(channel, latent_dim)
        self.xy_encoder = Encoder_xy(4, channel, latent_dim)
        self.x_encoder = Encoder_x(3, channel, latent_dim)

    def kl_divergence(self, posterior_latent_space, prior_latent_space):
        kl_div = kl.kl_divergence(posterior_latent_space, prior_latent_space)
        return kl_div

    def reparametrize(self, mu, logvar):
        std = logvar.mul(0.5).exp_()
        eps = torch.cuda.FloatTensor(std.size()).normal_()
        eps = Variable(eps)
        return eps.mul(std).add_(mu)

    def forward(self, x, y=None, training=True):
        if training:
            self.posterior, muxy, logvarxy = self.xy_encoder(torch.cat((x, y), 1))
            self.prior, mux, logvarx = self.x_encoder(x)
            lattent_loss = torch.mean(self.kl_divergence(self.posterior, self.prior))
            z_noise_post = self.reparametrize(muxy, logvarxy)
            z_noise_prior = self.reparametrize(mux, logvarx)
            self.sal_init_post, self.sal_ref_post = self.sal_encoder(x, z_noise_post)
            self.sal_init_prior, self.sal_ref_prior = self.sal_encoder(x, z_noise_prior)
            self.sal_init_post = F.upsample(self.sal_init_post, size=(x.shape[2], x.shape[3]), mode='bilinear',
                                            align_corners=True)
            self.sal_ref_post = F.upsample(self.sal_ref_post, size=(x.shape[2], x.shape[3]), mode='bilinear',
                                           align_corners=True)
            self.sal_init_prior = F.upsample(self.sal_init_prior, size=(x.shape[2], x.shape[3]), mode='bilinear',
                                            align_corners=True)
            self.sal_ref_prior = F.upsample(self.sal_ref_prior, size=(x.shape[2], x.shape[3]), mode='bilinear',
                                           align_corners=True)
            return self.sal_init_post, self.sal_ref_post, self.sal_init_prior, self.sal_ref_prior, lattent_loss
        else:
            _, mux, logvarx = self.x_encoder(x)
            z_noise = self.reparametrize(mux, logvarx)
            _, self.prob_pred = self.sal_encoder(x, z_noise)
            return self.prob_pred

class CAM_Module(nn.Module):
    """ Channel attention module"""
    # paper: Dual Attention Network for Scene Segmentation
    def __init__(self,in_dim):
        super(CAM_Module, self).__init__()
        self.chanel_in = in_dim
        self.gamma = Parameter(torch.zeros(1))
        self.softmax  = Softmax(dim=-1)
    def forward(self,x):
        """
            inputs :
                x : input feature maps( B X C X H X W)
            returns :
                out : attention value + input feature ( B X C X H X W)
                attention: B X C X C
        """
        m_batchsize, C, height, width = x.size()
        proj_query = x.view(m_batchsize, C, -1)
        proj_key = x.view(m_batchsize, C, -1).permute(0, 2, 1)
        energy = torch.bmm(proj_query, proj_key)
        energy_new = torch.max(energy, -1, keepdim=True)[0].expand_as(energy)-energy
        attention = self.softmax(energy_new)
        proj_value = x.view(m_batchsize, C, -1)

        out = torch.bmm(attention, proj_value)
        out = out.view(m_batchsize, C, height, width)

        out = self.gamma*out + x
        return out

class PAM_Module(nn.Module):
    """ Position attention module"""
    #paper: Dual Attention Network for Scene Segmentation
    def __init__(self, in_dim):
        super(PAM_Module, self).__init__()
        self.chanel_in = in_dim

        self.query_conv = nn.Conv2d(in_channels=in_dim, out_channels=in_dim//8, kernel_size=1)
        self.key_conv = nn.Conv2d(in_channels=in_dim, out_channels=in_dim//8, kernel_size=1)
        self.value_conv = nn.Conv2d(in_channels=in_dim, out_channels=in_dim, kernel_size=1)
        self.gamma = Parameter(torch.zeros(1))
        self.softmax = Softmax(dim=-1)

    def forward(self, x):
        """
            inputs :
                x : input feature maps( B X C X H X W)
            returns :
                out : attention value + input feature ( B X C X H X W)
                attention: B X (HxW) X (HxW)
        """
        m_batchsize, C, height, width = x.size()
        proj_query = self.query_conv(x).view(m_batchsize, -1, width*height).permute(0, 2, 1)
        proj_key = self.key_conv(x).view(m_batchsize, -1, width*height)
        energy = torch.bmm(proj_query, proj_key)
        attention = self.softmax(energy)
        proj_value = self.value_conv(x).view(m_batchsize, -1, width*height)

        out = torch.bmm(proj_value, attention.permute(0, 2, 1))
        out = out.view(m_batchsize, C, height, width)

        out = self.gamma*out + x
        return out

class Classifier_Module(nn.Module):
    def __init__(self,dilation_series,padding_series,NoLabels, input_channel):
        super(Classifier_Module, self).__init__()
        self.conv2d_list = nn.ModuleList()
        for dilation,padding in zip(dilation_series,padding_series):
            self.conv2d_list.append(nn.Conv2d(input_channel,NoLabels,kernel_size=3,stride=1, padding =padding, dilation = dilation,bias = True))
        for m in self.conv2d_list:
            m.weight.data.normal_(0, 0.01)

    def forward(self, x):
        out = self.conv2d_list[0](x)
        for i in range(len(self.conv2d_list)-1):
            out += self.conv2d_list[i+1](x)
        return out

## Channel Attention (CA) Layer
class CALayer(nn.Module):
    def __init__(self, channel, reduction=16):
        super(CALayer, self).__init__()
        # global average pooling: feature --> point
        self.avg_pool = nn.AdaptiveAvgPool2d(1)
        # feature channel downscale and upscale --> channel weight
        self.conv_du = nn.Sequential(
                nn.Conv2d(channel, channel // reduction, 1, padding=0, bias=True),
                nn.ReLU(inplace=True),
                nn.Conv2d(channel // reduction, channel, 1, padding=0, bias=True),
                nn.Sigmoid()
        )

    def forward(self, x):
        y = self.avg_pool(x)
        y = self.conv_du(y)
        return x * y

## Residual Channel Attention Block (RCAB)
class RCAB(nn.Module):
    # paper: Image Super-Resolution Using Very DeepResidual Channel Attention Networks
    # input: B*C*H*W
    # output: B*C*H*W
    def __init__(
        self, n_feat, kernel_size=3, reduction=16,
        bias=True, bn=False, act=nn.ReLU(True), res_scale=1):

        super(RCAB, self).__init__()
        modules_body = []
        for i in range(2):
            modules_body.append(self.default_conv(n_feat, n_feat, kernel_size, bias=bias))
            if bn: modules_body.append(nn.BatchNorm2d(n_feat))
            if i == 0: modules_body.append(act)
        modules_body.append(CALayer(n_feat, reduction))
        self.body = nn.Sequential(*modules_body)
        self.res_scale = res_scale

    def default_conv(self, in_channels, out_channels, kernel_size, bias=True):
        return nn.Conv2d(in_channels, out_channels, kernel_size,padding=(kernel_size // 2), bias=bias)

    def forward(self, x):
        res = self.body(x)
        #res = self.body(x).mul(self.res_scale)
        res += x
        return res

class BasicConv2d(nn.Module):
    def __init__(self, in_planes, out_planes, kernel_size, stride=1, padding=0, dilation=1):
        super(BasicConv2d, self).__init__()
        self.conv_bn = nn.Sequential(
            nn.Conv2d(in_planes, out_planes,
                      kernel_size=kernel_size, stride=stride,
                      padding=padding, dilation=dilation, bias=False),
            nn.BatchNorm2d(out_planes)
        )

    def forward(self, x):
        x = self.conv_bn(x)
        return x

class Triple_Conv(nn.Module):
    def __init__(self, in_channel, out_channel):
        super(Triple_Conv, self).__init__()
        self.reduce = nn.Sequential(
            BasicConv2d(in_channel, out_channel, 1),
            BasicConv2d(out_channel, out_channel, 3, padding=1),
            BasicConv2d(out_channel, out_channel, 3, padding=1)
        )

    def forward(self, x):
        return self.reduce(x)

class Saliency_feat_encoder(nn.Module):
    # resnet based encoder decoder
    def __init__(self, channel, latent_dim):
        super(Saliency_feat_encoder, self).__init__()
        self.resnet = B2_ResNet()
        # self.resnet=res2net50_v1b_26w_4s(pretrained=True)
        # self.relu = nn.ReLU(inplace=True)
        self.upsample8 = nn.Upsample(scale_factor=8, mode='bilinear', align_corners=True)
        self.dropout = nn.Dropout(0.3)
        self.layer5 = self._make_pred_layer(Classifier_Module, [6, 12, 18, 24], [6, 12, 18, 24], channel, 2048)
        self.layer6 = self._make_pred_layer(Classifier_Module, [6, 12, 18, 24], [6, 12, 18, 24], 1, channel*3)

        # self.conv1 = nn.Conv2d(256, channel, kernel_size=3, padding=1)
        self.conv2_1 = nn.Conv2d(512, channel, kernel_size=1, padding=0)
        self.conv2_2 = nn.Conv2d(channel, channel, kernel_size=3, padding=1)
        self.conv2_3 = nn.Conv2d(channel, channel, kernel_size=3, padding=1)
        self.conv3_1 = nn.Conv2d(1024, channel, kernel_size=1, padding=0)
        self.conv3_2 = nn.Conv2d(channel, channel, kernel_size=3, padding=1)
        self.conv3_3 = nn.Conv2d(channel, channel, kernel_size=3, padding=1)
        self.conv4_1 = nn.Conv2d(2048, channel, kernel_size=1, padding=0)
        self.conv4_2 = nn.Conv2d(channel, channel, kernel_size=3, padding=1)
        self.conv4_3 = nn.Conv2d(channel, channel, kernel_size=3, padding=1)

        # self.conv1 = Triple_Conv(256, channel)
        # self.conv2 = Triple_Conv(512, channel)
        # self.conv3 = Triple_Conv(1024, channel)
        # self.conv4 = Triple_Conv(2048, channel)

        self.conv_feat = nn.Conv2d(32 * 5, channel, kernel_size=3, padding=1)
        self.upsample4 = nn.Upsample(scale_factor=4, mode='bilinear', align_corners=True)
        self.upsample2 = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)

        self.pam_attention5 = PAM_Module(channel)
        self.pam_attention4 = PAM_Module(channel)
        self.pam_attention3 = PAM_Module(channel)
        self.pam_attention2 = PAM_Module(channel)

        self.cam_attention4 = CAM_Module(channel)
        self.cam_attention3 = CAM_Module(channel)
        self.cam_attention2 = CAM_Module(channel)

        self.pam_attention1 = PAM_Module(channel)
        self.racb_layer = RCAB(channel * 4)

        self.conv4 = self._make_pred_layer(Classifier_Module, [3, 6, 12, 18], [3, 6, 12, 18], channel, 2048)
        self.conv3 = self._make_pred_layer(Classifier_Module, [3, 6, 12, 18], [3, 6, 12, 18], channel, 1024)
        self.conv2 = self._make_pred_layer(Classifier_Module, [3, 6, 12, 18], [3, 6, 12, 18], channel, 512)
        self.conv1 = self._make_pred_layer(Classifier_Module, [3, 6, 12, 18], [3, 6, 12, 18], channel, 256)

        self.racb_43 = RCAB(channel * 2)
        self.racb_432 = RCAB(channel * 3)
        self.racb_4321 = RCAB(channel * 4)

        self.conv43 = Triple_Conv(2*channel, channel)
        self.conv432 = Triple_Conv(3 * channel, channel)
        self.conv4321 = Triple_Conv(4 * channel, channel)

        self.HA = HA()
        self.conv4_2 = self._make_pred_layer(Classifier_Module, [3, 6, 12, 18], [3, 6, 12, 18], channel, 2048)
        self.conv3_2 = self._make_pred_layer(Classifier_Module, [3, 6, 12, 18], [3, 6, 12, 18], channel, 1024)
        self.conv2_2 = self._make_pred_layer(Classifier_Module, [3, 6, 12, 18], [3, 6, 12, 18], channel, 512)
        self.pam_attention4_2 = PAM_Module(channel)
        self.pam_attention3_2 = PAM_Module(channel)
        self.pam_attention2_2 = PAM_Module(channel)
        self.cam_attention4_2 = CAM_Module(channel)
        self.cam_attention3_2 = CAM_Module(channel)
        self.cam_attention2_2 = CAM_Module(channel)
        self.racb_43_2 = RCAB(channel * 2)
        self.racb_432_2 = RCAB(channel * 3)
        self.conv43_2 = Triple_Conv(2 * channel, channel)
        self.conv432_2 = Triple_Conv(3 * channel, channel)
        self.conv4321_2 = Triple_Conv(4 * channel, channel)
        self.spatial_axes = [2, 3]
        self.conv_depth1 = BasicConv2d(3 + latent_dim, 3, kernel_size=3, padding=1)
        self.layer7 = self._make_pred_layer(Classifier_Module, [6, 12, 18, 24], [6, 12, 18, 24], 1, channel * 4)

        if self.training:
            self.initialize_weights()

    def _make_pred_layer(self, block, dilation_series, padding_series, NoLabels, input_channel):
        return block(dilation_series, padding_series, NoLabels, input_channel)

    def tile(self, a, dim, n_tile):
        """
        This function is taken form PyTorch forum and mimics the behavior of tf.tile.
        Source: https://discuss.pytorch.org/t/how-to-tile-a-tensor/13853/3
        """
        init_dim = a.size(dim)
        repeat_idx = [1] * a.dim()
        repeat_idx[dim] = n_tile
        a = a.repeat(*(repeat_idx))
        order_index = torch.LongTensor(np.concatenate([init_dim * np.arange(n_tile) + i for i in range(init_dim)])).to(device)
        return torch.index_select(a, dim, order_index)

    def forward(self, x, z):
        z = torch.unsqueeze(z, 2)
        z = self.tile(z, 2, x.shape[self.spatial_axes[0]])
        z = torch.unsqueeze(z, 3)
        z = self.tile(z, 3, x.shape[self.spatial_axes[1]])
        x = torch.cat((x, z), 1)
        x = self.conv_depth1(x)
        x = self.resnet.conv1(x)
        x = self.resnet.bn1(x)
        x = self.resnet.relu(x)
        x = self.resnet.maxpool(x)
        x1 = self.resnet.layer1(x)  # 256 x 64 x 64
        x2 = self.resnet.layer2(x1)  # 512 x 32 x 32
        x3 = self.resnet.layer3_1(x2)  # 1024 x 16 x 16
        x4 = self.resnet.layer4_1(x3)  # 2048 x 8 x 8

        conv1_feat = self.conv1(x1)
        conv2_feat = self.conv2(x2)
        conv2_feat1 = self.pam_attention2(conv2_feat)
        conv2_feat2 = self.cam_attention2(conv2_feat)
        conv2_feat = conv2_feat1+conv2_feat2
        conv3_feat = self.conv3(x3)
        conv3_feat1 = self.pam_attention3(conv3_feat)
        conv3_feat2 = self.cam_attention3(conv3_feat)
        conv3_feat = conv3_feat1+conv3_feat2
        conv4_feat = self.conv4(x4)
        conv4_feat1 = self.pam_attention4(conv4_feat)
        conv4_feat2 = self.cam_attention4(conv4_feat)
        conv4_feat = conv4_feat1+conv4_feat2
        conv4_feat = self.upsample2(conv4_feat)

        conv43 = torch.cat((conv4_feat, conv3_feat),1)
        conv43 = self.racb_43(conv43)
        conv43 = self.conv43(conv43)

        conv43 = self.upsample2(conv43)
        conv432 = torch.cat((self.upsample2(conv4_feat), conv43, conv2_feat), 1)
        conv432 = self.racb_432(conv432)
        # conv432 = self.conv432(conv432)
        #
        # conv432 = self.upsample2(conv432)
        # conv4321 = torch.cat((self.upsample4(conv4_feat), self.upsample2(conv43), conv432, conv1_feat), 1)
        # conv4321 = self.racb_4321(conv4321)
        # conv4321 = self.conv4321(conv4321)

        sal_init = self.layer6(conv432)

        x2_2 = self.HA(sal_init.sigmoid(), x2)
        x3_2 = self.resnet.layer3_2(x2_2)  # 1024 x 16 x 16
        x4_2 = self.resnet.layer4_2(x3_2)  # 2048 x 8 x 8

        conv2_feat = self.conv2_2(x2_2)
        conv2_feat1 = self.pam_attention2_2(conv2_feat)
        conv2_feat2 = self.cam_attention2_2(conv2_feat)
        conv2_feat = conv2_feat1+conv2_feat2
        conv3_feat = self.conv3_2(x3_2)
        conv3_feat1 = self.pam_attention3_2(conv3_feat)
        conv3_feat2 = self.cam_attention3_2(conv3_feat)
        conv3_feat = conv3_feat1+conv3_feat2
        conv4_feat = self.conv4_2(x4_2)
        conv4_feat1 = self.pam_attention4_2(conv4_feat)
        conv4_feat2 = self.cam_attention4_2(conv4_feat)
        conv4_feat = conv4_feat1+conv4_feat2

        conv4_feat = self.upsample2(conv4_feat)

        conv43 = torch.cat((conv4_feat, conv3_feat), 1)
        conv43 = self.racb_43_2(conv43)
        conv43 = self.conv43_2(conv43)

        conv43 = self.upsample2(conv43)
        conv432 = torch.cat((self.upsample2(conv4_feat), conv43, conv2_feat), 1)
        conv432 = self.racb_432_2(conv432)

        conv432 = self.conv432_2(conv432)

        conv432 = self.upsample2(conv432)
        conv4321 = torch.cat((self.upsample4(conv4_feat), self.upsample2(conv43), conv432, conv1_feat), 1)
        conv4321 = self.racb_4321(conv4321)
        sal_ref = self.layer7(conv4321)

        return self.upsample8(sal_init), self.upsample4(sal_ref)

    def initialize_weights(self):
        res50 = models.resnet50(pretrained=True)
        pretrained_dict = res50.state_dict()
        all_params = {}
        for k, v in self.resnet.state_dict().items():
            if k in pretrained_dict.keys():
                v = pretrained_dict[k]
                all_params[k] = v
            elif '_1' in k:
                name = k.split('_1')[0] + k.split('_1')[1]
                v = pretrained_dict[name]
                all_params[k] = v
            elif '_2' in k:
                name = k.split('_2')[0] + k.split('_2')[1]
                v = pretrained_dict[name]
                all_params[k] = v
        assert len(all_params.keys()) == len(self.resnet.state_dict().keys())
        self.resnet.load_state_dict(all_params)


# -*- coding: utf-8 -*-
# @Time    : 2020/5/12 15:06
# @Author  : 张开
# File      : 5. 文件操作.py

import time

from selenium import webdriver

driver = webdriver.Chrome()

driver.get('http://127.0.0.1:8022/upload/1')

time.sleep(1)
driver.find_element_by_id('ajaxFile').send_keys(r'D:\video\s28-testing-day17-接口自动化平台-实现-4\note\dengxin\data\接口测试示例-2.xlsx')
time.sleep(1)
driver.find_element_by_id('ajaxBtn').click()

time.sleep(5)
driver.quit()

选修进阶/python自动化测试/自动化/day18/note/5. 文件操作.py


import sys
# sys.path.append(r"D:\CK_Dev\Maya_Dev\Tool_Dev\module")
# from Qt import QtWidgets, QtCore, QtGui
# from Qt.QtCompat import loadUi
from PySide2 import QtWidgets, QtCore, QtGui
from MadGoatToolkit import Ui_MainDialog

class MadGoatTookkitMain(QtWidgets.QDialog, Ui_MainDialog):
    def __init__(self, parent=None):
        super(MadGoatTookkitMain, self).__init__(parent)
        self.setupUi(self)

    def closeEvent(self, event):
        self.MyFavorate.saveFavorite()

if __name__ == "__main__":
    app = QtWidgets.QApplication([])
    # widget = loadUi(ui_path)
    widget = MadGoatTookkitMain()
    widget.show()
    app.exec_()


"""
@description:主模块
"""
from dotenv import load_dotenv

# 加载 .env 文件中的环境变量

load_dotenv()

import os
import logging
from logging import handlers
import uvicorn
from fastapi import FastAPI, Request, status, WebSocket
from fastapi.responses import HTMLResponse
from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles
from fastapi.openapi.docs import (
    get_swagger_ui_html,
    get_swagger_ui_oauth2_redirect_html,
)
from fastapi_pagination import add_pagination
from starlette.exceptions import HTTPException as StarletteHTTPException
from fastapi.exceptions import RequestValidationError
from apps.chat.router import router as user_router, websocket_llm_handler
from apps.chat.admin_router import router as admin_router
from utils.custom_response import util_response
from utils.custom_config import BASE_DIR

ENV = os.environ.get('ENV_PROFILE')

# 主程序
app = FastAPI(
    title="FastAPI结构示例",  # 文档标题
    description="使用FastAPI实现基础的用户验证，权限管理，增删改查，文件上传下载等功能. 🚀",  # 文档简介
    version="0.0.1",  # 文档版本号
    docs_url=None, redoc_url=None,  # 配置离线文档
)

# 配置日志记录【记录 uvicorn 的日志】
@app.on_event("startup")
async def startup_event():
    log_path = os.path.join(BASE_DIR, 'logs')
    if not os.path.exists(log_path):
        os.mkdir(log_path)
    format_str = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")  # 设置日志格式
    logger = logging.getLogger("uvicorn.access")  # 获取日志对象
    # sh = logging.StreamHandler()  # 往屏幕上输出
    rh = handlers.RotatingFileHandler("logs/uvicorn.log", maxBytes=100 * 1024, backupCount=7, encoding='utf-8')  # 写入文件
    # sh.setFormatter(format_str)  # 指定格式
    rh.setFormatter(format_str)  # 指定格式
    # logger.addHandler(sh)
    logger.addHandler(rh)

# 跨域设置
app.add_middleware(
    CORSMiddleware,
    allow_origins=['*'],  # 允许跨域请求的源列表。
    allow_credentials=True,  # 指示跨域请求支持 cookies。
    allow_methods=["*"],  # 允许跨域请求的 HTTP 方法列表。
    allow_headers=["*"],  # 允许跨域请求的 HTTP 请求头列表。
)

# 挂载静态文件
app.mount("/static", StaticFiles(directory="static"), name="static")
# app.mount("/media", StaticFiles(directory="media"), name="media")

# 离线文档配置-由于资源文档经常请求不到，这里搞离线文档
@app.get("/docs", include_in_schema=False)
async def custom_swagger_ui_html():
    return get_swagger_ui_html(
        openapi_url=app.openapi_url,
        title=app.title + " - Swagger UI",
        oauth2_redirect_url=app.swagger_ui_oauth2_redirect_url,
        swagger_js_url="/static/doc/swagger-ui-bundle.js",
        swagger_css_url="/static/doc/swagger-ui.css",
    )

@app.get(app.swagger_ui_oauth2_redirect_url, include_in_schema=False)
async def swagger_ui_redirect():
    return get_swagger_ui_oauth2_redirect_html()

# 覆盖 HTTPException 错误处理器
@app.exception_handler(StarletteHTTPException)
async def http_exception_handler(request, exc):
    return util_response(http_status=exc.status_code, msg=str(exc.detail))

# 覆盖默认异常处理器[RequestValidationError是 Pydantic 的ValidationError的子类。]
@app.exception_handler(RequestValidationError)
async def validation_exception_handler(request, exc):
    return util_response(http_status=status.HTTP_400_BAD_REQUEST, msg=str(exc))

# todo 这里先这样定义全局异常返回，待优化
@app.middleware('http')
async def db_session_middleware(request: Request, call_next):
    response = util_response(http_status=status.HTTP_500_INTERNAL_SERVER_ERROR,
                             msg='Internal server error')
    # finally:
    #     return response
    # if ENV=='dev':
    #     try:
    #         response = await call_next(request)
    #     except Exception as e:
    #         response = util_response(http_status=status.HTTP_500_INTERNAL_SERVER_ERROR,
    #                              msg=f'Internal server error:{str(e)}')
    #         # msg=f'Internal server error:{str(e)}')
    #         logging.error(str(e))
    #         print(str(e))
    # elif ENV=='prod':
    #     response = await call_next(request)
    response = await call_next(request)
    return response

# 路由注册
app.include_router(user_router)
app.include_router(admin_router)

@app.get("/", tags=['测试'], deprecated=True)
async def root():
    return util_response(data='fast_api!')

@app.websocket("/ws/chat")
async def websocket_endpoint(websocket: WebSocket, x_current_user_id: int):
    await websocket_llm_handler(websocket, x_current_user_id)

add_pagination(app)

html = """
<!DOCTYPE html>

        <title>Chat</title>

WebSocket Chat

Your ID:

        <form action="" onsubmit="sendMessage(event)">
            <input type="text" id="messageText" autocomplete="off"/>
            <button>Send</button>
        </form>

"""

@app.get("/ws-html")
async def get():
    return HTMLResponse(html)

if __name__ == '__main__':
    uvicorn.run("main:app", host='0.0.0.0', port=8002, reload=True, log_config='uvicorn_config.json')  # 可以直接运行此脚本


import serial, os, random, time, sys

ser = serial.Serial('/dev/ttyUSB0',9600)
ser.flushInput()

#ringtones = os.listdir('ringtones')

while True:
    time.sleep(0.05)
    try:
        msgStr = ser.read(ser.inWaiting()).strip()
        # print(msgStr, type(msgStr)) -> bytes
        if len(msgStr) > 0:
            msg = int(msgStr)
            # print(msg)
        else:
            msg = -1
        if msg > 0 :
            print( "Ding dong", msg )
            #os.system("aplay ringtones/" + random.choice(ringtones))
            os.system(random.choice(["aplay ringtones/doorbell-four.wav","aplay ringtones/doorbell-six.wav"]))
            print('completed')
    except BaseException as err:
        print("Error...", type(err))
    sys.stdout.flush()


import logging
import os
import subprocess
from concurrent.futures.thread import ThreadPoolExecutor

import cyvcf2
from django.conf import settings
from django.core.cache import cache

from library.genomics.vcf_utils import vcf_get_ref_alt_end
from library.utils.file_utils import name_from_filename, mk_path
from snpdb import variant_collection
from snpdb.models import VariantCoordinate
from snpdb.variant_pk_lookup import VariantPKLookup
from upload.models import UploadStep, UploadStepTaskType, VCFPipelineStage
from upload.tasks.vcf.import_vcf_step_task import ImportVCFStepTask
from upload.vcf import sql_copy_files
from variantgrid.celery import app

USE_THREADS = True

def handle_unknown_variants(upload_pipeline, unknown_variants, unknown_variants_filename):
    sql_copy_files.write_sql_copy_csv(unknown_variants, unknown_variants_filename)

    name = UploadStep.CREATE_UNKNOWN_LOCI_AND_VARIANTS_TASK_NAME
    sort_order = upload_pipeline.get_max_step_sort_order() + 1
    upload_step = UploadStep.objects.create(upload_pipeline=upload_pipeline,
                                            name=name,
                                            sort_order=sort_order,
                                            task_type=UploadStepTaskType.CELERY,
                                            pipeline_stage=VCFPipelineStage.INSERT_UNKNOWN_VARIANTS,
                                            input_filename=unknown_variants_filename,
                                            items_to_process=len(unknown_variants))
    upload_step.launch_task(InsertUnknownVariantsTask)

class BulkUnknownVariantInserter:

    def __init__(self, upload_step):
        self.upload_step = upload_step
        self.unknown_dir = upload_step.upload_pipeline.get_pipeline_processing_subdir("vcf_unknown")
        self.rows_processed = 0
        self.unknown_variants_batch_id = 0
        self.executor = ThreadPoolExecutor(max_workers=1)
        self.child_futures = []
        self.variant_pk_lookup = VariantPKLookup(upload_step.genome_build)

    def process_vcf_record(self, variant):
        # Pre-processed by vcf_filter_unknown_contigs so only recognised contigs present
        # This has been decomposed (only be 1 alt per line)
        ref, alt, end = vcf_get_ref_alt_end(variant)
        variant_coordinate = VariantCoordinate(chrom=variant.CHROM, start=variant.POS, end=end, ref=ref, alt=alt)
        self.variant_pk_lookup.add(variant_coordinate)
        self.batch_process_check()

    def finish(self):
        self.batch_process_check(insert_all=True)

        # Make sure all child threads returned ok
        for f in self.child_futures:
            f.result()

    def batch_process_check(self, insert_all=False):
        if insert_all:
            minimum_size = 0
        else:
            minimum_size = settings.SQL_BATCH_INSERT_SIZE

        self.variant_pk_lookup.batch_check(minimum_size)

        # Unknown variants
        num_unknown_variants = len(self.variant_pk_lookup.unknown_variant_coordinates)
        if num_unknown_variants and num_unknown_variants >= minimum_size:
            uv_basename = f"unknown_variants_step_{self.upload_step.pk}_batch_{self.unknown_variants_batch_id}.csv"
            unknown_variants_filename = os.path.join(self.unknown_dir, uv_basename)
            huv_args = (self.upload_step.upload_pipeline, self.variant_pk_lookup.unknown_variant_coordinates,
                        unknown_variants_filename)

            if USE_THREADS:
                future = self.executor.submit(handle_unknown_variants, *huv_args)
                self.child_futures.append(future)
            else:
                handle_unknown_variants(*huv_args)

            self.variant_pk_lookup.clear()
            self.unknown_variants_batch_id += 1

class SeparateUnknownVariantsTask(ImportVCFStepTask):
    """ This is run on split VCF file.
        If it finds unknown, it writes files and launches InsertUnknownVariantsTask on them """

    def process_items(self, upload_step):
        bulk_inserter = BulkUnknownVariantInserter(upload_step)

        vcf_reader = cyvcf2.VCF(upload_step.input_filename)
        for v in vcf_reader:
            bulk_inserter.process_vcf_record(v)

        bulk_inserter.finish()  # Any leftovers
        variant_collection.count = bulk_inserter.rows_processed
        return bulk_inserter.rows_processed

class AnnotateImportedVCFTask(ImportVCFStepTask):
    """ Indexes then annotates the VCF """
    def process_items(self, upload_step):
        cmd_index = ['bcftools', 'index', upload_step.input_filename]
        subprocess.run(cmd_index, capture_output=True, check=True)

        common_settings = settings.VCF_IMPORT_COMMON_FILTERS.get(upload_step.genome_build.name)
        gnomad_af_filename = common_settings["gnomad_af_filename"]
        if not gnomad_af_filename.startswith("/"):
            gnomad_af_filename = os.path.join(settings.ANNOTATION_VEP_BASE_DIR, gnomad_af_filename)
        cmd_annotate = [
            "bcftools", "annotate", "-force",
            "--annotations", gnomad_af_filename, "--columns", "AF",
            "--output-type", "z", "--output", upload_step.output_filename,
            upload_step.input_filename
        ]
        subprocess.run(cmd_annotate, capture_output=True, check=True)
        return 1

class InsertUnknownVariantsTask(ImportVCFStepTask):
    """ Loci/Variants may have been inserted since the unknown variants were written, so we need to check
        in this process (which runs under a lock and in variant_id_single_workers so will avoid race conditions """

    def process_items(self, upload_step):
        LOCK_EXPIRE = 60 * 5  # 5 minutes
        LOCK_ID = "insert-unknown-variants-lock"

        if not settings.UPLOAD_ENABLED:
            raise ValueError(f"Uploads disabled, this should not have been called!")

        if cache.add(LOCK_ID, "true", LOCK_EXPIRE):  # fails if already exists
            try:
                return InsertUnknownVariantsTask.process_items_in_lock(upload_step)
            finally:
                logging.info("Releasing %s", LOCK_ID)
                cache.delete(LOCK_ID)
        else:
            logging.info("Someone else is doing the import")

    @staticmethod
    def process_items_in_lock(upload_step):
        logging.info("InsertUnknownVariantsTask.process_items() - %s", upload_step)

        # TODO: Fix this, bit of a hack at the moment....
        input_filename = upload_step.input_filename
        logging.info("file: %s", input_filename)

        pipeline_processing_dir = upload_step.upload_pipeline.get_pipeline_processing_dir()
        working_dir = os.path.join(pipeline_processing_dir, "unknown_variants", name_from_filename(input_filename))
        mk_path(working_dir)

        variant_pk_lookup = VariantPKLookup(upload_step.genome_build, working_dir=working_dir)

        with open(input_filename) as f:
            items_processed = 0
            # Python CSV reader dies with extremely long lines, so we just do by hand (not quoted or anything)
            for line in f:
                chrom, start, end, ref, alt = line.strip().split(",")  # Not quoted, exactly 5 columns
                variant_coordinate = VariantCoordinate(chrom=chrom, start=int(start), end=int(end), ref=ref, alt=alt)
                variant_pk_lookup.add(variant_coordinate)
                variant_pk_lookup.batch_check(settings.SQL_BATCH_INSERT_SIZE, insert_unknown=True)
                items_processed += 1
            # Any remaining
            variant_pk_lookup.batch_check(insert_unknown=True)
        return items_processed

SeparateUnknownVariantsTask = app.register_task(SeparateUnknownVariantsTask())
AnnotateImportedVCFTask = app.register_task(AnnotateImportedVCFTask())
InsertUnknownVariantsTask = app.register_task(InsertUnknownVariantsTask())


import numpy as np
import torch
import torch.nn as nn
from typing import Tuple, List, Union, Callable, Type, Iterator
from enum import Enum, unique
import torch.optim.optimizer

N = type(None)
V = np.array
VS = Union[Tuple[V, ...], List[V]]
VN = Union[V, N]
VNS = Union[VS, N]
T = torch.Tensor
TS = Union[Tuple[T, ...], List[T]]
TN = Union[T, N]
TNS = Union[TS, N]

V_Mesh = Tuple[V, V]
T_Mesh = Tuple[T, T]
D = torch.device
CPU = torch.device('cpu')
CUDA = lambda x: torch.device(f'cuda:{x}')

Optimizer = torch.optim.Adam
NoiseT = Union[int, Tuple[Union[bool, int], ...]]

@unique
class NoiseType(Enum):
    ALL_ZEROS = 0
    RANDOM = 1


import json
import random
import unittest
import requests
import responses

def random_cat_fact():
    response = requests.get('https://cat-fact.herokuapp.com/facts').json()
    facts = [fact['text'] for fact in response['all']]
    return random.choice(facts)

class RandomCatFactsTestCase(unittest.TestCase):

    @responses.activate
    #@mock.patch("requests.get")
    def test_random_cat_fact(self):
        facts = ["A cat has five toes on his front paws"]
        content = {
            'all': [{'text': fact} for fact in facts]
        }
        responses.add(
            responses.GET,
            'https://cat-fact.herokuapp.com/facts',
            body=json.dumps(content),
            content_type='application/json')

        fact = random_cat_fact()
        self.assertIn(fact, facts)

if __name__ == "__main__":
    unittest.main()


"""Commands module for EVENTS UTILITIES.
Access: Role-based"""

import asyncio
from typing import List

import discord

from petal.checks import all_checks, Messages
from petal.commands import core, shared
from petal.exceptions import CommandOperationError
from petal.menu import Menu
from petal.types import Src
from petal.util.embeds import Color
from petal.util.questions import ChatReply

class CommandsEvent(core.Commands):
    auth_fail = "This command requires the `{role}` role."
    role = "xPostRole"

    async def cmd_shiftcategory(
            self, args, src: Src, **_):
        return_channel: discord.TextChannel = src.channel
        event_group_id = self.config.get("eventCategory")
        main_chat_id = self.config.get("mainChatCategory")
        default_position = self.config.get("defaultEventChannelPosition")
        event_group: discord.CategoryChannel = self.client.main_guild.get_channel(event_group_id)
        main_chat: discord.CategoryChannel = self.client.main_guild.get_channel(main_chat_id)
        self.log.info(f"Main Chat position is: {main_chat.position}")
        self.log.info(f"Current event chat position is: {event_group.position}")

        if event_group.position > main_chat.position:
            await event_group.move(before=main_chat)
            await return_channel.send(content=f"{event_group.name} was moved right above {main_chat.name}")
        else:
            await event_group.edit(position=default_position)
            await return_channel.send(content=f"{event_group.name} was moved back to its default position ({default_position})")

    async def cmd_event(
        self, src, _image: str = None, _message: str = None, _nomenu: bool = False, **_
    ):
        """Post a message announcing the start of an event.

        Define a message which will be sent to one or more predefined channels. The message may include mass pings by way of including tags `{{e}}` and `{{h}}` for substitution.
        Destination channels may be selected conversationally or by way of a reaction-based menu.

        Options:
        `--message=<msg>` :: Define the message to send ahead of time. This will skip the step where Petal asks you what you want the message to say.
        `--nomenu` :: Forsake the Reaction UI and determine destination channels conversationally.
        """
        channels_list = []
        channels_dict = {}
        msg = ""
        for chan in self.config.get("xPostList"):
            channel = self.client.get_channel(chan)
            if channel is not None:
                msg += (
                    f"{len(channels_list)}. ({channel.name} [{channel.guild.name}])\n"
                )
                channels_list.append(channel)
                channels_dict[channel.guild.name + "/#" + channel.name] = channel
            else:
                self.log.warn(
                    f"{chan} is not a valid channel. I'd remove it if I were you."
                )

        # Get channels to send to.
        if _nomenu:
            # Do it only conversationally.
            menu = None
            while True:
                await self.client.send_message(
                    src.author,
                    src.channel,
                    "Hi there, "
                    + src.author.name
                    + "! Please select the number of "
                    + "each guild you want to post "
                    + "to. (dont separate the numbers)",
                )

                await self.client.send_message(src.author, src.channel, msg)

                chans = await Messages.waitfor(
                    self.client,
                    all_checks(
                        Messages.by_user(src.author), Messages.in_channel(src.channel)
                    ),
                    timeout=20,
                )

                if chans is None:
                    raise CommandOperationError(
                        "Sorry, the request timed out. Please make sure you"
                        " type a valid sequence of numbers."
                    )
                if self.validate_channel(channels_list, chans.content):
                    break
                else:
                    await self.client.send_message(
                        src.author,
                        src.channel,
                        "Invalid channel choices. You may try again immediately.",
                    )
            post_to = []
            for i in chans.content:
                print(channels_list[int(i)])
                post_to.append(channels_list[int(i)])
        else:
            # Use the Reaction-based GUI.
            menu = Menu(self.client, src.channel, "Event Announcement", user=src.author)
            if _image:
                menu.em.set_thumbnail(url=_image)
            selection = await menu.get_multi(
                list(channels_dict), title="Available Channels"
            )
            post_to = [channels_dict[c] for c in selection if c in channels_dict]
            if not post_to:
                menu.add_section("No Channels selected; Cancelled.", "Target Channels")
                await menu.post()
                return
            menu.add_section("\n".join(c.mention for c in post_to), "Target Channels")
            await menu.post()

        msgstr = yield ChatReply(
            "What do you want to send? (remember: {e} = `@ev` and {h} = `@here`)",
            120,
        )
        if not msgstr:
            # No reply.
            raise CommandOperationError("Text Input timed out.")

        try:
            msgstr = msgstr.format(e="@everyone", h="@here")
        except Exception as e:
            # Given a bad Format tag.
            raise CommandOperationError(
                "Could not perform `format()`. Make sure you did the braces"
                " around any `{e}`s or `{h}`s correctly."
                "\n(Any other braces in the message should be {{doubled}}.)"
            ) from e

        if _nomenu:
            embed = discord.Embed(
                title="Message to post", description=msgstr, colour=Color.info
            )
            embed.add_field(
                name="Channels", value="\n".join(c.mention for c in post_to)
            )
            await self.client.embed(src.channel, embed)

            msg2 = await Messages.waitfor(
                self.client,
                all_checks(
                    Messages.by_user(src.author), Messages.in_channel(src.channel)
                ),
                timeout=20,
                channel=src.channel,
                prompt="If this is correct, type `confirm`.",
            )

            if msg2 is None:
                raise CommandOperationError("Event post timed out.")
            elif msg2.content.lower() != "confirm":
                raise CommandOperationError("Event post cancelled.")
        else:
            menu.add_section(msgstr, "Confirm Message")
            await menu.post()
            proceed = await menu.get_bool()
            if proceed is None:
                menu.add_section("Posting timed out.", "Confirmation")
                await menu.post()
                return
            elif proceed is not True:
                menu.add_section("Posting cancelled.", "Confirmation")
                await menu.post()
                return

        posted: List[discord.Message] = []
        # TODO
        # em = discord.Embed()
        # if _image:
        #     em.set_image(url=_image)
        for i in post_to:
            posted.append(await i.send(msgstr))
            await asyncio.sleep(1)

        if menu:
            menu.add_section("Messages have been posted.", "Confirmation")
            await menu.post()
        else:
            yield "Messages have been posted."

        try:
            subkey, subname = self.get_event_subscription(msgstr)
        except AttributeError:
            pass
        else:
            if subkey is None:
                yield (
                    "I was unable to auto-detect any game titles in your post."
                    " No subscribers will be notified for this event."
                )
            else:
                n = await Messages.waitfor(
                    self.client,
                    all_checks(
                        Messages.by_user(src.author), Messages.in_channel(src.channel)
                    ),
                    timeout=20,
                    channel=src.channel,
                    prompt=f"I auto-detected a possible game in your announcement:"
                    f" **{subname}**. Would you like to notify subscribers? [y/N]",
                )

                if not n:
                    yield "Timed out."
                elif n.content.lower() not in ("y", "yes"):
                    yield "Subscribers will not be notified."
                else:
                    response = await self.notify_subscribers(
                        src.channel, posted[0], subkey
                    )
                    todelete = f"[{subkey}]"
                    for post in posted:
                        content = post.content
                        if todelete in content:
                            content = content.replace(todelete, "")
                            await post.edit(content=content)

                    yield response

    cmd_send = shared.factory_send(
        {
            "events": {"colour": 0x1B6649, "title": "Event Announcement"},
            "staff": {"colour": 0x4CCDDF, "title": "Staff Signal"},
        },
        "events",
    )

# Keep the actual classname unique from this common identifier
# Might make debugging nicer
CommandModule = CommandsEvent


import gdata.photos.service
import gdata.media
import gdata.geo

from django.core.cache import cache

class Album:
    def __init__(self, entry, albumId=None):
        self.title = entry.title.text
        self.url = entry.GetHtmlLink()
        self.date = entry.timestamp.datetime()
        if (albumId):
            self.id = albumId
            self.description = entry.subtitle.text
        else:
            self.id = entry.GetAlbumId()
            if (entry.media.thumbnail.count > 0):
                self.thumb = entry.media.thumbnail[0].url

class Photo:
    def __init__(self, entry=None):
        if not entry:
            return

        if (entry.media.thumbnail.count > 0):
            self.thumb = entry.media.thumbnail[0].url

        self.id = entry.gphoto_id.text

def clear_cache():
    cache.delete('albums')
    cache.delete('albums_home')

def get_picasa_feed(path):
    gd_client = gdata.photos.service.PhotosService()
    return gd_client.GetFeed(path)

def get_albums_for_url(url, key):
    albums = cache.get(key)

    if not albums:
        feed = get_picasa_feed(url)
        albums = []
        for entry in feed.entry:
            albums.append(Album(entry))
        cache.set(key, albums)

    return albums

def get_home_albums(user):
    return get_albums_for_url('/data/feed/api/user/%s?kind=album&max-results=100&thumbsize=144c' % user,
                              'albums_home')

def get_albums(user):
    return get_albums_for_url('/data/feed/api/user/%s?kind=album&max-results=100&thumbsize=144c' % user,
                              'albums')

def get_album(user, albumId):
    feed = get_picasa_feed('/data/feed/api/user/%s/albumid/%s?kind=photo&thumbsize=104c' %
                           (user, albumId));
    album = Album(feed, albumId)
    album.thumb = feed.icon.text
    photos = []
    for entry in feed.entry:
        photos.append(Photo(entry))

    album.photos = photos
    return album

def get_photo(user, albumId, photoId):
    feed = get_picasa_feed('/data/feed/api/user/%s/albumid/%s/photoid/%s?imgmax=576' %
                              (user, albumId, photoId));

    photo = Photo()
    photo.url = feed.media.content[0].url;
    photo.id = feed.gphoto_id
    photo.caption = feed.subtitle.text

    photo.album = get_album(user, albumId)

    return photo


from dataclasses import dataclass
from typing import List

from dataclasses_json import dataclass_json

@dataclass_json
@dataclass
class Bar:
    chords: List[str] = None
    beats: int = 4
    division: int = 4
    short: bool = False

    def add_chord(self, chord: str):
        if not self.chords:
            self.chords = list()
        self.chords.append(chord)

    def set_signature(self, beats, division):
        self.beats = beats
        self.division = division

@dataclass_json
@dataclass
class Part:
    bars: List[Bar] = None
    note: str = ""

    def add_bar(self, bar: Bar):
        if not self.bars:
            self.bars = list()

        self.bars.append(bar)

@dataclass_json
@dataclass
class Element:
    name: str
    parts: List[Part] = None
    note: str = ""

    def add_part(self, part: Part):
        if not self.parts:
            self.parts = list()

        self.parts.append(part)

@dataclass_json
@dataclass
class Song:
    title: str
    key: str
    elements: List[Element] = None
    beats: int = 4
    division: int = 4

    def add_element(self, element: Element):
        if not self.elements:
            self.elements = list()

        self.elements.append(element)

    def set_signature(self, beats, division):
        self.beats = beats
        self.division = division


import pygame
import pygame as pg
import numpy as np
from pygame.locals import *
from escenario import *

class Cartel(pygame.sprite.Sprite):
    #_________________________________________________inicializador
    def __init__(self, x, y, width, height,fuente,numero_de_fotogramas,tasa_fotogramas,instante_de_partida):
        self.letra=""
        #tamaño
        self.x = x
        self.y = y
        #posicion
        self.width = width
        self.height = height
        self.fuente = fuente
        self.numero_de_fotogramas = numero_de_fotogramas
        self.tasa_fotogramas = tasa_fotogramas
        self.instante_de_partida = instante_de_partida
        pygame.font.init()
        temp = pygame.font.Font(None,45)
        self.texto = temp.render(self.letra,0,(0,0,0))
        self.textooriginal=None
        self.textoArmado=None
    #_________________________________________________Cambiar iamgen de boton
    def update_image(self, ui):
        img = pygame.image.load(ui)
        self.image1 = img
        img2 = pg.transform.scale(self.image1, (self.width, self.height))
        self.image2 = img2
    #_________________________________________________Ajustar tamaño de boton
    def update_size(self, width, height):
        self.width = width
        self.height = height
        img2 = pg.transform.scale(self.image1, (self.width, self.height))
        self.image2 = img2
    #_________________________________________________Dibujar Boton
    def draw(self,buff):
        buff.blit(self.image2, (self.x, self.y))
        buff.blit(self.texto,(self.x+50,self.y+50))

    def iniciar_contador(self,screen):
        segundos_totales = self.instante_de_partida - (self.numero_de_fotogramas // self.tasa_fotogramas)
        if segundos_totales < 0:
            segundos_totales = 0
        minutos = segundos_totales // 60
        segundos = segundos_totales % 60
        texto_de_salida = "Tiempo: {0:02}:{1:02}".format(minutos, segundos)
        texto = self.fuente.render(texto_de_salida, True, (0, 0, 0))
        screen.blit(texto, [self.x+60, self.y+20])
        self.numero_de_fotogramas += 1
    def aumentar_palabra(self,temp):
        self.letra+=temp
        pygame.font.init()
        temp = pygame.font.Font(None,30)
        self.texto = temp.render(self.letra,0,(0,0,0))


import os
import sys
import json

# ----------------------------------------------------------------------------------------------------------------------

class Fabric():

    def __init__(self, zfm_dir):
        self.config_dir = zfm_dir

        #
        # Node placeholder.
        #
        self.nodes = {}
        self.gcids = set()

    def _write_type(self, data, filename):
        filename = os.path.join(self.config_dir, filename)

        try:
            os.makedirs(self.config_dir, exist_ok=True)
            with open(filename, 'w') as f:
                json.dump(data, f,indent=4, separators=(",", ": "))
        except:
            print(sys.exc_info())
            sys.exit(1)

    def add_node(self,name,node):
        self.nodes[name] = node
        if node.node_type() != 'Switch':
            self.gcids |= set(node.GCIDs())

    def find_node(self,name):
        return self.nodes[name]

    def write(self):

        #
        # Bundle up the nodes so that we can write them as JSON.
        #
        switch_data  = {}
        compute_data = {}
        memory_data  = {}
        io_data      = {}

        for name,node in self.nodes.items():
            node_type = node.node_type()
            if   node_type == 'Switch'  : switch_data[name]  = node.configuration()
            elif node_type == 'Compute' : compute_data[name] = node.configuration()
            elif node_type == 'IO'      : io_data[name]      = node.configuration()
            elif node_type == 'Memory'  : memory_data[name]  = node.configuration()

        #
        # Create the directories and write the profiles for the nodes.
        #
        self._write_type(switch_data,  'switch_nodes.conf')
        self._write_type(compute_data, 'compute_nodes.conf')
        self._write_type(io_data,      'io_nodes.conf')
        self._write_type(memory_data,  'memory_nodes.conf')

# ----------------------------------------------------------------------------------------------------------------------


from datetime import datetime

from flask import Blueprint, request, session

from ..api import BJUTLabAPI
from ..utilities import (
    login_required,
    get_validate_param,
    post_validate_param,
    parse_date_str,
    TIME_FORMAT,
    Validator,
    TEACHER_ID_FORMAT
)

ExpBP = Blueprint('Experiment', __name__, url_prefix='/Experiment')
api = BJUTLabAPI()
ACCEPTABLE_ORDER_TYPE = [0]

@ExpBP.route('/orders', methods=['GET'])
@login_required
def get_orders():
    return api.exp.get_orders(session['type'], session['id'])

@ExpBP.route('/order', methods=['GET'])
@login_required
def get_order():
    order_id = get_validate_param(request.args, 'order_id', int, Validator.digit_in_range, ((1, None),))

    return api.exp.get_order(order_id, session['type'], session['id'])

@ExpBP.route('/order', methods=['POST'])
@login_required
def create_order():
    form = request.form
    commit = post_validate_param(form, 'commit', Validator.datetime_in_range,
                                 (TIME_FORMAT, datetime.now(), (0, 5 * 60)))
    use = post_validate_param(form, 'use')
    time_range = post_validate_param(form, 'time_range', Validator.string_format,
                                     (r'^\d{1,2}:\d{2}~\d{1,2}:\d{2}$',))
    lab_id = post_validate_param(form, 'lab_id', Validator.isdigit)
    usage = post_validate_param(form, 'usage', Validator.string_length, ((None, 200),))
    commit_dt = datetime.strptime(commit, TIME_FORMAT)
    use_d = parse_date_str('use', use)
    instruments = request.values.getlist('instruments[]')
    return api.exp.create_order(session['id'], session['type'], commit_dt, use_d,
                                time_range, lab_id, usage, instruments)

@ExpBP.route('/labs', methods=['GET'])
def get_labs():
    args = request.args
    filter_str = get_validate_param(args, 'filter', str, Validator.string_format, (
        r'^(name->\w{1,30}|principal->{' + TEACHER_ID_FORMAT +
        r'}|open->[01]|time->\d{2}:\d{2}~\d{2}:\d{2}|day->[1-7]{1,7})$',), True)
    return api.exp.get_labs(filter_str)

@ExpBP.route('/lab/<int:lab_id>', methods=['GET'])
def get_lab(lab_id: int):
    ok, err = Validator.digit_in_range(('lab_id', lab_id, (1, None)))
    if not ok:
        raise err
    return api.exp.get_lab(lab_id)


import pandas as pd
import warnings
import matplotlib.pyplot as plt
from et_xmlfile import xmlfile
from matplotlib import font_manager, rc

font_path = './D2Coding-Ver1.3.2-20180524.ttf'
font_name = font_manager.FontProperties(fname=font_path).get_name()
rc('font', family=font_name)

df = pd.read_excel('./남북한발전전력량.xlsx')
df = pd.DataFrame(df.fillna(method='ffill'))
df.rename(columns={'전력량 (억㎾h)': '국가', '발전 전력별': '전력별'}, inplace=True)
df.set_index('국가', inplace=True)
print(df)
print()

south = pd.DataFrame(df.loc['남한'])
north = pd.DataFrame(df.loc['북한'])
print(south)
print(north)
print()

south.reset_index(inplace=True)
south.drop('국가', axis=1, inplace=True)
south.set_index('전력별', inplace=True)
south.replace({'-': 0}, inplace=True)
# south = south.loc['합계']

north.reset_index(inplace=True)
north.drop('국가', axis=1, inplace=True)
north.set_index('전력별', inplace=True)
north.replace({'-': 0}, inplace=True)
# north = north.loc['합계']

south.rename(index={'합계': '총발전량'}, inplace=True)
north.rename(index={'합계': '총발전량'}, inplace=True)

south = south.loc['총발전량']
north = north.loc['총발전량']
print(south)
print(north)
print()

# 스타일 지정
plt.style.use('ggplot')
plt.rcParams['axes.unicode_minus'] = False   # 마이너스 부호 출력 설정

# ax = north.plot(kind='area', stacked=True, alpha=0.2, figsize=(20, 10))
ax = south.plot(kind='area', stacked=False, alpha=0.2, figsize=(20, 10))
ax.set_title('대한민국 총발전량', size=30, color='brown', weight='bold')
ax.set_xlabel('기간', size=20, color='blue')
ax.set_ylabel('발전량', size=20, color='blue')
ax.legend(loc='best', fontsize=15)

plt.show()

ax = north.plot(kind='area', stacked=False, alpha=0.2, figsize=(20, 10))
ax.set_title('북조선 총발전량', size=30, color='brown', weight='bold')
ax.set_xlabel('기간', size=20, color='blue')
ax.set_ylabel('발전량', size=20, color='blue')
ax.legend(loc='best', fontsize=15)

plt.show()


import os
import tempfile

from avocado import Test
from avocado.utils import archive
from avocado.utils import process
from avocado.utils import build, distro
from avocado.utils.software_manager.manager import SoftwareManager

class Lmbench(Test):

    """
    lmbench is a series of micro benchmarks intended to measure basic
    operating system and hardware system metrics. The benchmarks fall
    into three general classes: bandwidth, latency, and ``other''.
    """

    def setUp(self):
        '''
        Build lmbench
        Source:
        http://www.bitmover.com/lmbench/lmbench3.tar.gz
        '''
        fsdir = self.params.get('fsdir', default=None)
        temp_file = self.params.get('temp_file', default=None)
        memory_size_mb = self.params.get('MB', default=125)
        self.tmpdir = tempfile.mkdtemp(prefix='avocado_' + __name__)
        smm = SoftwareManager()
        detected_distro = distro.detect()
        packages = ['gcc', 'make', 'patch']
        if detected_distro.name in ['SuSE', 'rhel', 'centos', 'fedora']:
            packages.extend(['libtirpc-devel'])
        elif detected_distro.name == "Ubuntu":
            packages.extend(['libtirpc-dev'])
        for package in packages:
            if not smm.check_installed(package) and not smm.install(package):
                self.cancel("%s is needed for the test to be run" % package)
        tarball = self.fetch_asset('http://www.bitmover.com'
                                   '/lmbench/lmbench3.tar.gz')
        archive.extract(tarball, self.workdir)
        version = os.path.basename(tarball.split('.tar.')[0])
        self.sourcedir = os.path.join(self.workdir, version)

        # Patch for lmbench

        os.chdir(self.sourcedir)

        makefile_patch = 'patch -p1 < %s' % self.get_data('makefile.patch')
        build_patch = 'patch -p1 < %s' % self.get_data(
            '0001-Fix-build-issues-with-lmbench.patch')
        lmbench_fix_patch = 'patch -p1 < %s' % self.get_data(
            '0002-Changing-shebangs-on-lmbench-scripts.patch')
        ostype_fix_patch = 'patch -p1 < %s' % self.get_data(
            'fix_add_os_type.patch')
        lsleek_fix_patch = 'patch -p1 < %s' % self.get_data(
            'fix_lseek.patch')

        process.run(makefile_patch, shell=True)
        process.run(build_patch, shell=True)
        process.run(lmbench_fix_patch, shell=True)
        process.run(ostype_fix_patch, shell=True)
        process.run(lsleek_fix_patch, shell=True)

        build.make(self.sourcedir, extra_args='LDFLAGS=-ltirpc')

        # configure lmbench
        os.chdir(self.sourcedir)

        process.system('yes "" | make config', shell=True, ignore_status=True)

        # find the lmbench config file
        output = os.popen('ls -1 bin/*/CONFIG*').read()
        config_files = output.splitlines()
        if len(config_files) != 1:
            self.error('Config not found : % s' % config_files)
        config_file = config_files[0]
        if not fsdir:
            fsdir = self.tmpdir
        if not temp_file:
            temp_file = os.path.join(self.tmpdir, 'XXX')

        # patch the resulted config to use the proper temporary directory and
        # file locations
        with open(config_file, "r+") as cfg_file:
            lines = cfg_file.readlines()
            cfg_file.seek(0)
            cfg_file.truncate()
            for line in lines:
                if line.startswith("FSDIR="):
                    cfg_file.write("FSDIR=%s\n" % fsdir)
                elif line.startswith("FILE="):
                    cfg_file.write("FILE=%s\n" % temp_file)
                elif line.startswith("MB="):
                    cfg_file.write("MB=%s\n" % memory_size_mb)
                else:
                    cfg_file.write(line)
            # Printing the config file
            cfg_file.seek(0)
            for line in cfg_file.readlines():
                print(line)

    def test(self):

        os.chdir(self.sourcedir)
        build.make(self.sourcedir, extra_args='rerun')
        build.make(self.sourcedir, extra_args='rerun')
        build.make(self.sourcedir, extra_args='see')


import os
import os.path as osp
import six
import shutil
import unittest
import glob
import gwgen
import numpy as np
import tempfile
from gwgen.main import GWGENOrganizer
import gwgen.utils as utils
from model_organization.config import setup_logging

test_root = osp.abspath(osp.dirname(__file__))

_test_stations = osp.join(test_root, 'test_stations.dat')

_short_test_stations = osp.join(test_root, 'test_stations_short.dat')

_long_test_stations = osp.join(test_root, 'test_stations_long.dat')

_eecra_test_stations = osp.join(test_root, 'eecra_test_stations.dat')

_short_eecra_test_stations = osp.join(test_root,
                                      'eecra_test_stations_short.dat')

setup_logging(osp.join(test_root, 'logging.yaml'))

on_travis = os.getenv('TRAVIS')

db_config = dict(
    database='travis_ci_test')

class BaseTest(unittest.TestCase):
    """Test the :class:`gwgen.main.ModuleOrganizer` class"""

    test_dir = None

    remove_at_cleanup = True

    use_db = True

    stations_type = 'normal'

    #: number of cores to use
    nprocs = None

    #: Boolean that is True, if the tests should be run in serial
    serial = False

    _station_files = {
        'normal': _test_stations,
        'short': _short_test_stations,
        'long': _long_test_stations,
        }

    _eecra_station_files = {
        'normal': _eecra_test_stations,
        'short': _short_eecra_test_stations,
        }

    @classmethod
    def setUpClass(cls):
        # try to connect to a postgres database
        if cls.use_db:
            try:
                utils.get_postgres_engine(create=True, test=True, **db_config)
                cls.use_db = True
            except:
                cls.use_db = False

    def setUp(self):
        from psyplot import rcParams
        self.test_dir = tempfile.mkdtemp(prefix='tmp_gwgentest')
        os.environ['GWGENCONFIGDIR'] = self.config_dir = osp.join(
            self.test_dir, 'config')
        if not osp.exists(self.test_dir):
            os.makedirs(self.test_dir)
        if not osp.exists(self.config_dir):
            os.makedirs(self.config_dir)
        self.test_db = osp.basename(self.test_dir)
        src_station_file = self._station_files[self.stations_type]
        self.stations_file = osp.join(self.test_dir,
                                      osp.basename(src_station_file))
        shutil.copyfile(src_station_file, self.stations_file)

        src_eecra_file = self._eecra_station_files[self.stations_type]
        self.eecra_stations_file = osp.join(self.test_dir,
                                            osp.basename(src_eecra_file))
        shutil.copyfile(src_eecra_file, self.eecra_stations_file)
        self.organizer = GWGENOrganizer()
        global_conf = self.organizer.config.global_config
        global_conf['data'] = osp.join(test_root, 'test_data')
        global_conf['use_relative_links'] = False
        global_conf['nprocs'] = self.nprocs or 'all'
        global_conf['serial'] = self.serial
        if self.use_db:
            self._clear_db()
            global_conf.update(db_config)
        rcParams['plotter.linreg.nboot'] = 1

    def tearDown(self):
        from psyplot import rcParams
        if self.remove_at_cleanup:
            if osp.exists(self.test_dir):
                shutil.rmtree(self.test_dir)
            if osp.exists(self.config_dir):
                shutil.rmtree(self.config_dir)
            if self.use_db:
                self._clear_db()
        rcParams.update_from_defaultParams(plotters=False)

        del self.organizer
        del self.test_dir
        del self.config_dir

    def _clear_db(self):
        engine = utils.get_postgres_engine(db_config['database'])[0]
        conn = engine.connect()
        for table in engine.table_names():
            conn.execute("DROP TABLE %s;" % table)
        conn.close()

    @property
    def stations(self):
        """A numpy array of the stations in :attr:`test_stations`"""
        return np.loadtxt(
            self.stations_file, dtype='S11', usecols=[0]).astype(
                np.str_)

    def _test_setup(self):
        """Test the setup of a project. We make this method private such that
        it is not called everytime"""
        self.organizer.setup(self.test_dir, 'test_project0', link=False)
        mpath = osp.join(self.test_dir, 'test_project0')
        self.assertTrue(osp.isdir(mpath))
        original_files = sorted(map(osp.basename, glob.glob(osp.join(
            osp.dirname(gwgen.__file__), 'src', '*.f90'))))
        copied_files = sorted(map(osp.basename, glob.glob(osp.join(
            mpath, 'src', '*.f90'))))
        self.assertEqual(original_files, copied_files)
        self.assertIn('test_project0', self.organizer.config.projects)

        # createa new project and let it automatically assign the name
        self.organizer.setup(self.test_dir)
        mpath = osp.join(self.test_dir, 'test_project1')
        self.assertTrue(osp.isdir(mpath))
        original_files = sorted(map(osp.basename, glob.glob(osp.join(
            osp.dirname(gwgen.__file__), 'src', '*.f90'))))
        copied_files = sorted(map(osp.basename, glob.glob(osp.join(
            mpath, 'src', '*.f90'))))
        self.assertEqual(original_files, copied_files)
        self.assertIn('test_project1', self.organizer.config.projects)

    def _test_init(self):
        """Test the intialization of a new experiment. We make this method
        private such that it is not called everytime"""
        self.organizer.setup(self.test_dir)
        projectname = self.organizer.projectname
        self.organizer.init(experiment='testexp0')
        expdir = osp.join(
            self.test_dir, projectname, 'experiments', 'testexp0')
        self.assertTrue(osp.exists(expdir),
                        msg='Experiment directory %s does not exist!' % expdir)
        self.assertIn('testexp0', self.organizer.config.experiments)

        # test without argument
        self.organizer.setup(self.test_dir)
        projectname = self.organizer.projectname
        self.organizer.init(experiment=None)
        expdir = osp.join(
            self.test_dir, projectname, 'experiments', 'testexp1')
        self.assertTrue(osp.exists(expdir),
                        msg='Experiment directory %s does not exist!' % expdir)
        self.assertIn('testexp1', self.organizer.config.experiments)

    @staticmethod
    def _test_url(url, *args, **kwargs):
        if six.PY3:
            from urllib import request
            request.urlopen(url, *args, **kwargs)
        else:
            import urllib
            urllib.urlopen(url, *args, **kwargs)

    def assertAlmostArrayEqual(self, actual, desired, rtol=1e-07, atol=0,
                               msg=None, **kwargs):
        """Asserts that the two given arrays are almost the same

        This method uses the :func:`numpy.testing.assert_allclose` function
        to compare the two given arrays.

        Parameters
        ----------
        actual : array_like
            Array obtained.
        desired : array_like
            Array desired.
        rtol : float, optional
            Relative tolerance.
        atol : float, optional
            Absolute tolerance.
        equal_nan : bool, optional.
            If True, NaNs will compare equal.
        err_msg : str, optional
            The error message to be printed in case of failure.
        verbose : bool, optional
            If True, the conflicting values are appended to the error message.
        """
        try:
            np.testing.assert_allclose(actual, desired, rtol=rtol, atol=atol,
                                       err_msg=msg or '', **kwargs)
        except AssertionError as e:
            if six.PY2:
                self.fail(e.message)
            else:
                self.fail(str(e))

# check if we are online by trying to connect to google
try:
    BaseTest._test_url('https://www.google.de')
    online = True
except:
    online = False

{'rcParams': 'psyplot.rcParams', 'request': 'urllib.request', 'urllib': 'urllib'}._test_url


from .preprocess import read_pcap, preprocess_df
import xgboost as xgb

def preprocess_data(folder_path):
    df = read_pcap(folder_path)
    done_df = preprocess_df(df)
    # done_df = done_df.loc[:, 'sport':'chksum']
    return done_df

def predict_models(apps_model_path, classes_model_path, df):
    dict_apps = {'vpn_bittorrent': 0, 'vpn_email2a': 1, 'vpn_facebook_audio2': 2, 'vpn_facebook_chat1a': 3, 'vpn_ftps_A': 4, 'vpn_hangouts_audio1': 5, 'vpn_hangouts_chat1a': 6, 'vpn_icq_chat1a': 7, 'vpn_netflix_A': 8, 'vpn_sftp_A': 9, 'vpn_skype_audio1': 10, 'vpn_skype_chat1a': 11, 'vpn_skype_files1a': 12, 'vpn_spotify_A': 13, 'vpn_vimeo_A': 14, 'vpn_voipbuster1a': 15, 'vpn_youtube_A': 16}
    dict_classes = {'chats': 0, 'voice_chats': 1, 'video': 2, 'audio': 3, 'files': 4, 'email': 5, 'sftp': 6, 'ftps': 7}
    # apps_model = xgb.XGBClassifier()
    # apps_model.load_model(apps_model_path)
    done_df = df.copy()
    # done_df['apps_class'] = [dict_apps.keys()[dict_apps.values().index(i)] for i in apps_model.predict(df)]

    classes_model = xgb.XGBClassifier()
    classes_model.load_model(classes_model_path)
    done_df['class2'] = classes_model.predict(df.loc[:, 'sport':'chksum'])

    print(len(done_df['class2']), len(done_df.loc[done_df['class2']==0]))
    # done_df['classes_model'] = [dict_classes.keys()[dict_apps.values().index(i)] for i in classes_model.predict(df)]
    return done_df

def save_csv(path, pcap_dir):
    done_df  = predict_models('saved_models/apps_model.txt', '/app/ml/saved_models/classes_model.txt', preprocess_data(pcap_dir))
    done_df.to_csv(path)


import json
import argparse
import logging.config
import time
import re
from sys import argv
from datetime import date, datetime
from dataclasses import dataclass, field
from collections import defaultdict
from socrata import *
from utils import *
from storage import *

logger = logging.getLogger(__name__)

with open('phase_2/config.json', 'r') as f:
    config = json.load(f)

parser = argparse.ArgumentParser(description="Extract the dataset named Police Department Incident Reports: 2018 to Present \
                                 from The City and Condado of San Francisco. Socrata Open Data API have been used to programmatically \
                                 return the dataset.",
                                 formatter_class=argparse.ArgumentDefaultsHelpFormatter)
type_modes = ["LAST_DATE", "REFRESH"]
parser.add_argument("-m", "--mode", type=str, choices=type_modes, default=type_modes[0], required=True, help="the reference is the last date")
# parser.add_argument("-s", "--start_date", type=datetime.fromisoformat, default="2018-01-01", required=("--mode="+type_modes[1] in argv), help="used with refresh mode: from the desired date since 2018-01-01")
# parser.add_argument("-e", "--end_date", type=datetime.fromisoformat, default="2018-01-01", required=("--mode="+type_modes[1] in argv), help="used with refresh mode: from the desired date since 2018-01-01")
args = vars(parser.parse_args())

@dataclass
class Data:
    args: "defaultdict[dict]" = field(default_factory=lambda: defaultdict(dict), init=False)
    blob_list_inter: list = field(default_factory=list, init=False)
    blob_dict: "defaultdict[dict]" = field(default_factory=lambda: defaultdict(dict), init=False)
    date_list: list = field(default_factory=list, init=False)
    date_dict: "defaultdict[dict]" = field(default_factory=lambda: defaultdict(dict), init=False)
    day_range: int = field(init=False)
    end_date: str = field(init=False)
    get_url: str = field(init=False)
    get_headers: str = field(init=False)
    params: str = field(init=False)
    start_date: str = field(init=False)

    def __post_init__(self):
        self.get_url = config.get('api').get('domain').get('url') + config.get('api').get('dataset').get('san_francisco_data') + '.json'
        self.get_headers = config.get('api').get('headers')
        self.params = config.get('api').get('params')

    def list_dates(self):
        self.blob_list = Blob().list_blobs()
        pattern = "([0-9]{4}\-[0-9]{2}\-[0-9]{2})"

        for date in self.blob_list:
            dates = str(re.findall(pattern=pattern, string=date)).replace('[','').replace(']','').strip("'")
            self.blob_list_inter.append((datetime.fromisoformat(str(dates)) + timedelta(days=1, hours=0,minutes=0,seconds=0)).strftime("%Y-%m-%dT%H:%M:%S"))
            max_date = max(self.blob_list_inter)

        self.blob_dict["last_date"] = max_date
        logger.info('From {cls} the attr: {attr}'.format(cls=type(self.blob_dict).__name__, attr=self.blob_dict), exc_info=True)
        return self.blob_dict["last_date"]

    def ingestion(self):
        if args["mode"] == 'LAST_DATE':
            max_date = Data().list_dates()
            request = Socrata(url=self.get_url, headers=self.get_headers, parameters=self.params, start_date=max_date).api_connection()

            logger.info('From {cls} the attr: {attr}'.format(cls=type(request).__name__, attr=request), exc_info=True)

            if request.status_code != 200:
                logger.info('From {cls} the attr: {attr}'.format(cls=type(request.status_code).__name__, attr=request.status_code), exc_info=True)
                raise RuntimeError('Can''t retrieve latest timestamp.' + request.text)
            else:
                json.dumps(request.json(), indent=4)
                logger.info('From {cls} the attr: {attr}'.format(cls=type(request.json()).__name__, attr=request.json()[-1:]), exc_info=True)

            blob_name = str(datetime.fromisoformat(max_date).strftime("%Y-%m-%d")) + "_dataset_" + str(config.get('api').get('dataset').get('san_francisco_data')) + "_crimes_in_loaded_" + str(time.time_ns()) + ".json"
            blob = Blob_obj(blob_name=blob_name).blob_obj()

            logger.info('From {cls} the attr: {attr}'.format(cls=type(blob).__name__, attr=blob), exc_info=True)

            #with blob.open("w") as write_file:
            #    json.dump(request.json(), write_file, indent=4)
            message = f'{max_date} loaded in the {Bucket().bucket_name} bucket'

            logger.info('From {cls} the attr: {attr}'.format(cls=type(message).__name__, attr=message), exc_info=True)

        return request.json()

if __name__ == '__main__':
        Data