#!/usr/bin/env python3
'''
Namespace for the ICNS format.
@see https://en.wikipedia.org/wiki/Apple_Icon_Image_format
'''
import os  # path
from typing import Union, Optional, Tuple, Iterator, List, Iterable, Set
from . import PackBytes, RawData

class CanNotDetermine(Exception):
    pass

class Media:
    KeyT = Union[str, bytes]
    __slots__ = ['key', 'types', 'size', 'channels', 'bits', 'availability',
                 'desc', 'compressable', 'retina', 'maxsize', 'ext_certain']

    def __init__(
        self,
        key: KeyT,
        types: List[str],
        size: Union[int, Tuple[int, int], None] = None,
        *,
        ch: Optional[int] = None,
        bits: Optional[int] = None,
        os: Optional[float] = None,
        desc: str = '',
    ) -> None:
        self.key = key
        self.types = types
        self.size = (size, size) if isinstance(size, int) else size
        self.availability = os
        self.desc = desc
        # computed properties
        self.compressable = self.is_type('argb') or self.is_type('rgb')
        self.retina = '@2x' in self.desc
        if self.is_type('rgb'):
            ch = 3
            bits = 8
        if self.is_type('argb'):
            ch = 4
            bits = 8
        self.channels = ch
        self.bits = bits
        self.maxsize = None  # type: Optional[int]
        if self.size and ch and bits:
            self.maxsize = self.size[0] * self.size[1] * ch * bits // 8
        self.ext_certain = all(x in ['png', 'argb', 'plist', 'jp2', 'icns']
                               for x in self.types)

    def is_type(self, typ: str) -> bool:
        return typ in self.types

    def is_binary(self) -> bool:
        return any(x in self.types for x in ['rgb', 'bin'])

    def fallback_ext(self) -> str:
        if self.channels in [1, 2]:
            return self.desc  # guaranteed to be icon, mask, or iconmask
        return self.types[-1]

    def decompress(self, data: bytes, ext: Optional[str] = '-?-') \
            -> Optional[List[int]]:
        ''' Returns None if media is not decompressable. '''
        if self.compressable:
            if ext == '-?-':
                ext = RawData.determine_file_ext(data)
            if ext == 'argb':
                return PackBytes.unpack(data[4:])  # remove ARGB header
            if ext is None or ext == 'rgb':  # RGB files dont have magic number
                if self.key == 'it32':
                    data = data[4:]
                return PackBytes.unpack(data)
        return None

    def filename(self, *, key_only: bool = False, size_only: bool = False) \
            -> str:
        if key_only:
            if os.path.exists(__file__.upper()):  # check case senstive
                if self.key in ['sb24', 'icsb']:
                    return self.key + '-a'  # type: ignore
                elif self.key in ['SB24', 'icsB']:
                    return self.key + '-b'  # type: ignore
            return str(self.key)  # dont return directy, may be b''-str
        else:
            if self.is_type('icns'):
                return self.desc
            if not self.size:
                return str(self.key)  # dont return directy, may be b''-str
            w, h = self.size
            suffix = ''
            if self.retina:
                w //= 2
                h //= 2
                suffix = '@2x'
            if size_only:
                if self.bits == 1:
                    suffix += '-mono'
            else:
                if self.desc in ['icon', 'iconmask']:
                    suffix += '-icon{}b'.format(self.bits)
                if self.desc in ['mask', 'iconmask']:
                    suffix += '-mask{}b'.format(self.bits)
            return '{}x{}{}'.format(w, h, suffix)

    def __repr__(self) -> str:
        return '<{}: {}, {}.{}>'.format(
            type(self).__name__, str(self.key), self.filename(), self.types[0])

    def __str__(self) -> str:
        T = ''
        if self.size:
            T += '{}x{}, '.format(*self.size)
            if self.maxsize:
                T += '{}ch@{}-bit={}, '.format(
                    self.channels, self.bits, self.maxsize)
        if self.desc:
            T += self.desc + ', '
        return '{}: {}macOS {}+'.format(
            str(self.key), T, self.availability or '?')

_TYPES = {x.key: x for x in (
    # Read support for these:
    Media('ICON', ['bin'], 32, ch=1, bits=1, os=1.0, desc='icon'),
    Media('ICN#', ['bin'], 32, ch=2, bits=1, os=6.0, desc='iconmask'),
    Media('icm#', ['bin'], (16, 12), ch=2, bits=1, os=6.0, desc='iconmask'),
    Media('icm4', ['bin'], (16, 12), ch=1, bits=4, os=7.0, desc='icon'),
    Media('icm8', ['bin'], (16, 12), ch=1, bits=8, os=7.0, desc='icon'),
    Media('ics#', ['bin'], 16, ch=2, bits=1, os=6.0, desc='iconmask'),
    Media('ics4', ['bin'], 16, ch=1, bits=4, os=7.0, desc='icon'),
    Media('ics8', ['bin'], 16, ch=1, bits=8, os=7.0, desc='icon'),
    Media('is32', ['rgb'], 16, os=8.5),
    Media('s8mk', ['bin'], 16, ch=1, bits=8, os=8.5, desc='mask'),
    Media('icl4', ['bin'], 32, ch=1, bits=4, os=7.0, desc='icon'),
    Media('icl8', ['bin'], 32, ch=1, bits=8, os=7.0, desc='icon'),
    Media('il32', ['rgb'], 32, os=8.5),
    Media('l8mk', ['bin'], 32, ch=1, bits=8, os=8.5, desc='mask'),
    Media('ich#', ['bin'], 48, ch=2, bits=1, os=8.5, desc='iconmask'),
    Media('ich4', ['bin'], 48, ch=1, bits=4, os=8.5, desc='icon'),
    Media('ich8', ['bin'], 48, ch=1, bits=8, os=8.5, desc='icon'),
    Media('ih32', ['rgb'], 48, os=8.5),
    Media('h8mk', ['bin'], 48, ch=1, bits=8, os=8.5, desc='mask'),
    Media('it32', ['rgb'], 128, os=10.0),
    Media('t8mk', ['bin'], 128, ch=1, bits=8, os=10.0, desc='mask'),
    # Write support for these:
    Media('icp4', ['png', 'jp2', 'rgb'], 16, os=10.7),
    Media('icp5', ['png', 'jp2', 'rgb'], 32, os=10.7),
    Media('icp6', ['png', 'jp2'], 48, os=10.7),
    Media('ic07', ['png', 'jp2'], 128, os=10.7),
    Media('ic08', ['png', 'jp2'], 256, os=10.5),
    Media('ic09', ['png', 'jp2'], 512, os=10.5),
    Media('ic10', ['png', 'jp2'], 1024, os=10.7, desc='or 512x512@2x (10.8)'),
    Media('ic11', ['png', 'jp2'], 32, os=10.8, desc='16x16@2x'),
    Media('ic12', ['png', 'jp2'], 64, os=10.8, desc='32x32@2x'),
    Media('ic13', ['png', 'jp2'], 256, os=10.8, desc='128x128@2x'),
    Media('ic14', ['png', 'jp2'], 512, os=10.8, desc='256x256@2x'),
    Media('ic04', ['argb', 'png', 'jp2'], 16, os=11.0),  # ARGB is macOS 11+
    Media('ic05', ['argb', 'png', 'jp2'], 32, os=11.0),
    Media('icsb', ['argb', 'png', 'jp2'], 18, os=11.0),
    Media('icsB', ['png', 'jp2'], 36, desc='18x18@2x'),
    Media('sb24', ['png', 'jp2'], 24),
    Media('SB24', ['png', 'jp2'], 48, desc='24x24@2x'),
    # ICNS media files
    Media('sbtp', ['icns'], desc='template'),
    Media('slct', ['icns'], desc='selected'),
    Media(b'\xFD\xD9\x2F\xA8', ['icns'], os=10.14, desc='dark'),
    # Meta types:
    Media('TOC ', ['bin'], os=10.7, desc='Table of Contents'),
    Media('icnV', ['bin'], desc='4-byte Icon Composer.app bundle version'),
    Media('name', ['bin'], desc='Unknown'),
    Media('info', ['plist'], desc='Info binary plist'),
)}

def enum_img_mask_pairs(available_keys: Iterable[Media.KeyT]) \
        -> Iterator[Tuple[Optional[str], Optional[str]]]:
    for mask_k, *imgs in [  # list probably never changes, ARGB FTW
        ('s8mk', 'is32', 'ics8', 'ics4', 'icp4'),
        ('l8mk', 'il32', 'icl8', 'icl4', 'icp5'),
        ('h8mk', 'ih32', 'ich8', 'ich4'),
        ('t8mk', 'it32'),
    ]:
        mk = mask_k if mask_k in available_keys else None
        any_img = False
        for img_k in imgs:
            if img_k in available_keys:
                any_img = True
                yield img_k, mk
        if mk and not any_img:
            yield None, mk

def enum_png_convertable(available_keys: Iterable[Media.KeyT]) \
        -> Iterator[Tuple[Media.KeyT, Optional[Media.KeyT]]]:
    ''' Yield (image-key, mask-key or None) '''
    for img in _TYPES.values():
        if img.key not in available_keys:
            continue
        if img.is_type('argb') or img.bits == 1:  # allow mono icons
            yield img.key, None
        elif img.is_type('rgb'):
            mask_key = None
            for mask in _TYPES.values():
                if mask.key not in available_keys:
                    continue
                if mask.desc == 'mask' and mask.size == img.size:
                    mask_key = mask.key
                    break
            yield img.key, mask_key

def supported_extensions() -> Set[str]:
    return set(y for x in _TYPES.values() for y in x.types)

def get(key: Media.KeyT) -> Media:
    try:
        return _TYPES[key]
    except KeyError:
        pass
    raise NotImplementedError('Unsupported icns type "' + str(key) + '"')

def key_from_readable(key: str) -> Media.KeyT:
    key_mapping = {
        'dark': b'\xFD\xD9\x2F\xA8',
        'selected': 'slct',
        'template': 'sbtp',
        'toc': 'TOC ',
    }
    return key_mapping.get(key.lower(), key)  # type: ignore[return-value]

def match_maxsize(total: int, typ: str) -> Media:
    assert(typ == 'argb' or typ == 'rgb')
    ret = [x for x in _TYPES.values() if x.is_type(typ) and x.maxsize == total]
    return _best_option(ret, typ)

def guess(data: bytes, filename: Optional[str] = None) -> Media:
    '''
    Guess icns media type by analyzing the raw data + file naming convention.
    Use:
    - @2x.png or @2x.jp2 for retina images
    - directly name the file with the corresponding icns-key
    - or {selected|template|dark}.icns to select the proper icns key.
    '''
    # Set type directly via filename
    if filename:
        bname = os.path.splitext(os.path.basename(filename))[0]
        if bname in _TYPES:
            return _TYPES[bname]

    # Filter attributes
    desc = None
    size = None
    maxsize = None
    retina = False

    # Guess extension
    ext = RawData.determine_file_ext(data)
    if not ext and filename:
        if filename.endswith('.rgb'):
            ext = 'rgb'
        elif filename.endswith('.mask'):
            maxsize = len(data)
            desc = 'mask'

    # Guess image size
    if ext:
        size = RawData.determine_image_size(data, ext)

    # if filename is set, then bname is also set (see above)
    if filename:
        # Guess retina flag
        retina = bname.lower().endswith('@2x')
        # Guess icns-specific type
        if ext == 'icns':
            for candidate in ['template', 'selected', 'dark']:
                if bname.endswith(candidate):
                    desc = candidate
                    break

    # stupid double usage of ic10, enforce retina flag
    if size == (1024, 1024):
        retina = True

    choices = []
    for x in _TYPES.values():
        if retina != x.retina:  # png + jp2
            continue
        if desc and desc != x.desc:  # icns or rgb-mask
            continue
        if ext:
            if size != x.size or not x.is_type(ext):
                continue
        else:  # not ext
            if x.ext_certain:
                continue
            if maxsize and x.maxsize and maxsize != x.maxsize:  # mask only
                continue
        choices.append(x)

    return _best_option(choices, ext)

def _best_option(choices: List[Media], ext: Optional[str] = None) -> Media:
    '''
    Get most favorable media type.
    If more than one option exists, choose based on order index of ext.
    '''
    if len(choices) == 1:
        return choices[0]
    # Try get most favorable type (sort order of types)
    if ext:
        best_i = 99
        best_choice = []
        for x in choices:
            i = x.types.index(ext)
            if i < best_i:
                best_i = i
                best_choice = [x]
            elif i == best_i:
                best_choice.append(x)
        if len(best_choice) == 1:
            return best_choice[0]
        choices = best_choice

    raise CanNotDetermine('Could not determine type – one of {}.'.format(
        [x.key for x in choices]))


#!flask/bin/python
from flask import Flask, jsonify, request

from params import pool_fee
import wallet_json_rpc
import mining
import sqlite_handler

app = Flask(__name__)

last_account_transferred_block = 0

class BePatientError(Exception):
    pass

def transfer_account_handler(new_pubkey):
    """
    It will transfer a random empty account to the new_pubkey. One account per block is allowed.
    :param new_pubkey:
    :return:
    """
    global last_account_transferred_block

    try:
        current_block = wallet_json_rpc.get_current_block()
    except wallet_json_rpc.WalletCommError:
        raise wallet_json_rpc.WalletCommError

    if current_block == last_account_transferred_block:
        raise BePatientError

    try:
        acc_number = wallet_json_rpc.get_a_zero_balance_account_number()
    except wallet_json_rpc.WalletCommError:
        raise wallet_json_rpc.WalletCommError
    except wallet_json_rpc.NoEmptyAccountError:
        raise wallet_json_rpc.NoEmptyAccountError

    try:
        wallet_json_rpc.change_key(new_pubkey, acc_number)
    except wallet_json_rpc.WalletCommError:
        raise wallet_json_rpc.WalletCommError
    except wallet_json_rpc.InputParameterError:
        raise wallet_json_rpc.InputParameterError

    last_account_transferred_block = current_block
    return acc_number

@app.route('/pool_data', methods=['GET'])
def get_pool_data():
    network_height = wallet_json_rpc.get_current_block()
    pool_data = {
        "current_block": str(network_height),
        "net_hashrate": str(wallet_json_rpc.get_net_hashrate(network_height)) + " Gh",
        "algorithm": "Pascal",
        "poolhash": str(round(mining.get_pool_hr() / 10**9, 2)) + " Gh",
        "nethash":  0,
        "workers":  str(mining.No_miners()),
        "fee":      str(pool_fee) + "%",
        "period":   "Every block"
    }
    return jsonify({'pool_data': pool_data})

@app.route('/miner_data/<int:account>', methods=['GET'])
def get_miner_data(account):
    miner_data = {
        "account": str(account),
        "hashrate": str(round(mining.get_hr(account) / 10**9, 3)) + " Gh",
        "1hour":            0,
        "24hours":          0,
        "average_mined":    0,
        "payments":         sqlite_handler.db.get_account_payments(account)
    }
    return jsonify({'miner_data': miner_data})

@app.route('/get_account', methods=['POST'])
def get_account():
    pubkey = request.get_json(force=True)['pubkey']
    try:
        acc_number = transfer_account_handler(pubkey)
    except wallet_json_rpc.WalletCommError:
        return jsonify({'result': 'An error occured on the pool side. Please try again. If you see this message multiple times please report it to the team on Discord or Telegram. Thank you!'})
    except BePatientError:
        return jsonify({'result': 'An account was already sent to someone in this block. Please try again in a few minutes. If you see this message multiple times please report it to the team on Discord or Telegram. Thank you!'})
    except wallet_json_rpc.NoEmptyAccountError:
        return jsonify({'result': 'No free account is left on the pool. Please try again in a few minutes. If you see this message multiple times please report it to the team on Discord or Telegram. Thank you!'})
    except wallet_json_rpc.InputParameterError:
        return jsonify({'result': 'Wrong public key! You can export your public key from the wallet. It has to start with "3G". If you see this message multiple times please report it to the team on Discord or Telegram. Thank you!'})

    return jsonify({'result': 'Account ' + str(acc_number) + ' was successfully sent to your public key. You will see it in the next block in appr. 5 minutes.'})

def start_restapi():
    app.run(debug=False, port = 3000)


import json
import random

from flask import Flask, redirect, url_for, request, jsonify, make_response

app = Flask(__name__)

data = json.load(open('services.json'))
imeiservicelist = data["IMEISERVICELIST"]
accountinfo = data["ACCOUNTINFO"]

@app.route('/api/index.php', methods=['POST', 'GET'])
def imeiservices():  # put application's code here
    if request.method == 'POST' and request.form['action'] == 'imeiservicelist':
        return imeiservicelist
    elif request.method == 'POST' and request.form['action'] == 'accountinfo':
        return accountinfo
    else:
        return make_response(jsonify({"status": "read documentation"}), 400)

if __name__ == '__main__':
    app.run()


#!/usr/bin/env python

import argparse
import logging
import os
import shutil
import sys
import numpy as np
import time

import torch
import torch.nn as nn
import torch.optim as optim
from torch.distributions import MultivariateNormal
from torch.utils import tensorboard
from torch.utils.data import DataLoader
from torchvision import transforms
from torchvision.utils import save_image
from torchvision.transform import to_tensor
from torchvision.datasets import MNIST

################################################################################
# Utils
################################################################################
def setup_logger(name, save_dir, distributed_rank=0):
    logger = logging.getLogger(name)
    logger.setLevel(logging.DEBUG)
    # don't log results for the non-master process
    if distributed_rank > 0:
        return logger
    ch = logging.StreamHandler(stream=sys.stdout)
    ch.setLevel(logging.DEBUG)
    formatter = logging.Formatter("%(asctime)s %(name)s %(levelname)s: %(message)s")
    ch.setFormatter(formatter)
    logger.addHandler(ch)

    if save_dir:
        fh = logging.FileHandler(os.path.join(save_dir, "log.txt"))
        fh.setLevel(logging.DEBUG)
        fh.setFormatter(formatter)
        logger.addHandler(fh)

    return logger

def setup_exp(save_dir, exp_name, clean_run='False'):
    if not os.path.exists(save_dir):
        os.mkdir(save_dir)
    runs_dir = os.path.join(save_dir, 'runs', exp_name)
    chkpt_dir = os.path.join(save_dir, exp_name, 'chkpt')
    log_dir = os.path.join(save_dir, exp_name, 'logs')

    def mkdir(dir_name):
        print(dir_name)
        if not os.path.exists(dir_name):
            os.makedirs(dir_name)

    if clean_run:
        if os.path.exists(log_dir) or os.path.exists(chkpt_dir):
            shutil.rmtree(os.path.join(save_dir, exp_name))
        if os.path.exists(runs_dir):
            shutil.rmtree(runs_dir)
        time.sleep(5)
    mkdir(log_dir)
    mkdir(runs_dir)
    mkdir(chkpt_dir)

    return log_dir, runs_dir, chkpt_dir

################################################################################
# Model
################################################################################
class Model(nn.Module):
    def __init__(self, args):
        super(Model, self).__init__()
        self.enc_hid_sz = args.enc_hid_sz
        self.embed_sz = args.latent_sz

        self.enc_layer1 = nn.Linear(args.input_sz, 256)
        self.enc_layer2 = nn.Linear(256, 128)
        self.enc_layer3 = nn.Linear(128, 64)
        self.enc_layer_mu = nn.Linear(64, 20)
        self.enc_layer_var = nn.Linear(64, self.embed_sz)

        self.dec_layer1 = nn.Linear(self.embed_sz, 28)
        self.dec_layer2 = nn.Linear(28, 64)
        self.dec_layer3 = nn.Linear(64, 128)
        self.dec_layer4 = nn.Linear(128, 256)
        self.dec_layer_out = nn.Linear(256, args.input_sz)

    def enc_forward(self, x):
        x = nn.functional.relu(self.enc_layer1(x))
        x = nn.functional.relu(self.enc_layer2(x))
        x = nn.functional.relu(self.enc_layer3(x))
        mu = self.enc_layer_mu(x)
        logvar = self.enc_layer_var(x)
        return mu, logvar

    def sample(self, mu, logvar):
        eps = torch.randn(logvar.shape).to(logvar.device)
        return mu + eps * logvar

    def dec_forward(self, z):
        z = nn.functional.relu(self.dec_layer1(z))
        z = nn.functional.relu(self.dec_layer2(z))
        z = nn.functional.relu(self.dec_layer3(z))
        z = nn.functional.relu(self.dec_layer4(z))
        return nn.functional.sigmoid(self.dec_layer_out(z))

    def forward(self, x):
        mu, logvar = self.enc_forward(x)
        z = self.sample(mu, logvar)
        x_prime = self.dec_forward(z)
        return mu, logvar, z, x_prime

################################################################################
# Dataloader
################################################################################
def get_loader(args, train):
    # transform_fnc = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))])
    transform_fnc = transforms.Compose([transforms.ToTensor()])
    if train:
        loader = DataLoader(MNIST('./.torch/data', train=True, download=True, transform=transform_fnc),
                            batch_size=args.batch_size, shuffle=True)
    else:
        loader = DataLoader(MNIST('./.torch/data', train=False, download=True, transform=transform_fnc),
                            batch_size=args.batch_size, shuffle=True)

    return loader

################################################################################
# Trainer
################################################################################
def kl_criterion(mu, logvar, reduce=None):
    kl_loss = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp(), 1)
    return torch.mean(kl_loss)

def visualize(x, writer, writer_idx, args):
    img = x.view((-1,) + args.img_shape)
    img = torch.unsqueeze(img[:, :], 1)
    writer.add_images('Recon Output', img, writer_idx, dataformats='NCHW')

def latent_visualize(model, writer, writer_ix, args):
    a = torch.arange(0, 1, 10)
    b = torch.arange(0, 1, 10)
    res = torch.zeros(100, 28, 28)
    for i in range(0, 10):
        for j in range(0, 10):
            z = [i, j]
            res[i*10 + j] = model.dec_forward((z))

def train(args, model, loader, logger, writer, device):
    model.train()
    model = model.to(device)
    batch_size = args.batch_size
    num_batches = args.num_samples // batch_size
    optimizer = optim.Adam(params=model.parameters(), lr=args.lr)
    num_epochs = args.num_epochs
    # cls_criterion = nn.functional.binary_cross_entropy (reduction='mean')
    kl_factor = 1.0
    train_loader, test_loader = loader
    writer_idx = 0
    for eid in range(num_epochs):
        for bid, data in enumerate(train_loader):
            x, y = data
            x = x.view(min(x.shape[0], args.batch_size), -1).to(device)
            y = y.to(device)
            mu, logvar , z, x_prime = model.forward(x)
            kl_loss = kl_criterion(mu, logvar)
            # Important See the defination of bce loss at https://pytorch.org/docs/stable/nn.html#loss-functions
            cls_loss = torch.mean(torch.sum(nn.functional.binary_cross_entropy(x_prime.view(x.shape[0], -1), x.view(x.shape[0], -1), reduction='none'), 1))
            loss = cls_loss + kl_factor * kl_loss
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            if bid % args.log_freq == 0:
                writer_idx += 1
                logger.info(f'Writer Idx:{writer_idx%args.log_freq}, Epoch Idx:{eid}, cls_loss: {cls_loss}, kl_loss: {kl_loss}, Loss: {kl_factor * kl_loss + cls_loss}')
                writer.add_scalar('kl_loss', kl_loss, writer_idx)
                writer.add_scalar('cls_loss', cls_loss, writer_idx)
                writer.add_scalar('kl_factor', kl_factor, writer_idx)
                visualize(x_prime, writer, writer_idx, args)
        test(args, model, test_loader, logger, writer, device, eid, kl_factor)
        sample = torch.randn(64, args.latent_sz).to(device)
        res = model.dec_forward(sample)
        img = res.view((-1,) + args.img_shape)
        img = torch.unsqueeze(img[:, :], 1)
        save_image(img, 'final_ouput.png')
        writer.add_images('Final Output', img, eid, dataformats='NCHW')
        kl_factor = 1.0 / (1+np.exp(-eid+3))

def test(args, model, test_loader, logger, writer, device, eid, kl_factor):
    model.eval()
    model = model.to(device)
    batch_size = args.batch_size
    num_batches = args.test_num_samples // batch_size
    cls_criterion = nn.BCEWithLogitsLoss()
    for bid, data in enumerate(test_loader):
        writer_idx = bid * batch_size + (eid * num_batches * batch_size)
        x, y = data
        x = x.view(min(args.batch_size, x.shape[0]), -1).to(device)
        y = y.to(device)
        mu, logvar, z, x_prime = model.forward(x)
        kl_loss = kl_criterion(mu, logvar, reduce='mean')
        cls_loss = cls_criterion(x_prime, x)
        loss = cls_loss + kl_factor * kl_loss

    logger.info(f'Epoch: {eid}, Test Loss: {loss}')
    writer.add_scalar('test_kl_loss', kl_loss, writer_idx)
    writer.add_scalar('test_cls_loss', cls_loss, writer_idx)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--save_dir', help='Save Directory', default='./variational', type=str)
    parser.add_argument('--clean_run', help='Start From scratch', default=True, type=bool)
    parser.add_argument('--exp_name', help='Exp Name', default='default', type=str)
    parser.add_argument('--batch_size', help='Batch Size', default=64, type=int)
    parser.add_argument('--num_epochs', help='Number of Epochs', default=100, type=int)
    parser.add_argument('--num_samples', help='Number of samples to use', default=-1, type=int)
    parser.add_argument('--lr', help='learning rate', default=1e-4, type=float)
    parser.add_argument('--device_id', help='Device number', default=-1, type=int)
    parser.add_argument('--enc_hid_sz', help='Encoder Hidden Size', default=100, type=int)
    parser.add_argument('--latent_sz', help='Latent Embedding Size', default=20, type=int)
    parser.add_argument('--log_freq', help='Logging Frequency', default=100, type=int)
    parser.add_argument('--img_shape', help='Image Shape', default=(28, 28), type=tuple)
    parser.add_argument('--input_sz', help='Input Size', default=784, type=int)

    args = parser.parse_args()
    device = 'cuda:1' if torch.cuda.is_available() else 'cpu'
    log_dir, runs_dir, chkpt_dir = setup_exp(args.save_dir, args.exp_name, args.clean_run)
    writer = tensorboard.SummaryWriter(runs_dir)
    logger = setup_logger(args.exp_name, log_dir)
    model = Model(args)
    train_loader = get_loader(args, train=True)
    test_loader = get_loader(args, train=False)
    if args.num_samples == -1:
        args.num_samples = len(train_loader.dataset)
        args.test_num_samples = len(test_loader.dataset)
    train(args, model, [train_loader, test_loader], logger, writer, device)

if __name__ == '__main__':
    main()


#import relevant libraries (first streamlit)
import streamlit as st
import pandas as pd
import plotly as py
import plotly.graph_objs as pg
import plotly.express as px

st.title("Data Visualization Tutorial: Interactive Streamlit Dashboard")
st.title("Anandi Gupta")

"""
This tutorial provides users a step-by-step guide for creating their own interactive data visualization dashboards using Streamlit and Plotly.

Note, to insert text blocks as shown here, simply type your text within a doc string.
"""

## URLs - Our world in data github as an example
#Emissions
url = "https://raw.githubusercontent.com/owid/co2-data/master/owid-co2-data.csv"
#read in our world in data github
df_emissions = pd.read_csv(url)

df_emissions = df_emissions[['iso_code', 'country', 'year', 'co2', 'co2_per_capita', 'energy_per_gdp', 'energy_per_capita', 'population']]

#### Create relevant sections and corresponding plots using plotly

if st.sidebar.checkbox("Bar Charts Over Time", key="section1"):
    st.header("Example Bar Chart (Use st.header to include meaningful section header)")
    df_emissions_viz = df_emissions[df_emissions['country'] == "World"]
    df_emissions_viz = df_emissions_viz[df_emissions_viz['year'] > 1980].reset_index()
    fig = px.bar(df_emissions_viz, x=df_emissions_viz['year'], y=df_emissions_viz['co2'], labels={'co2':'Global CO2 Emissions', "year": "Year"},
                 title='CO2 Emissions Over Time')
    st.plotly_chart(fig, use_container_width=True)
    st.write("Use st.write to insert a meaningful takeaway")

if st.sidebar.checkbox("Chloropleth Maps", key="section2"):
    st.header("Example Chloropleth Map: Use st.header to include meaningful section header")

    answer1 = st.selectbox(label="Choose indicator",
    options=("CO2 Emissions Per Capita", "Energy Consumption Per Capita"))
    answer2 = st.slider('Choose time period', 2016, 2020, 2016)

    def cc_causes_maps(var):
        df_emissions2 = df_emissions[df_emissions['year'] == answer2]

        data = dict(type='choropleth',
                locations = df_emissions2['iso_code'],
                z = df_emissions2[var],
                text = df_emissions2['country'])

        layout = dict(title = answer1,
                  geo = dict(showframe = False,
                           projection = {'type':'robinson'},
                           showlakes = True,
                           lakecolor = 'rgb(0,191,255)'))
        x = pg.Figure(data = [data], layout = layout)
        st.plotly_chart(x, use_container_width=True)

    if answer1 == "CO2 Emissions Per Capita":
        cc_causes_maps('co2_per_capita')
        st.write("Use st.write to insert a meaningful takeaway about CO2 Emissions per Capita")

    elif answer1 == "Energy Consumption Per Capita":
        cc_causes_maps('energy_per_capita')
        st.write("Use st.write to insert a meaningful takeaway about Energy Consumption per Capita")

if st.sidebar.checkbox("Text Section", key="section6"):
    st.header("Use st.header to insert meaningful text header")
    """
    Insert any text within doc strings

    To add links use format "[NAME] (LINK)" as seen in the example below:

    1) [Anandi Gupta's Streamlit Dashboard](https://anandigupta-data-science-i-ds2-final-project-streamlit-2-fahu9f.streamlit.app/)
    """


from django.urls import path
from . import views
from django.conf import settings
from django.conf.urls.static import static

urlpatterns = [
    path('', views.home, name = 'home'),
    path('projects/', views.projects, name = 'projects'),
    path('testimonials/', views.testimonials, name = 'testimonials'),
    path('contact/', views.contact, name = 'contact'),
    path('projects/<int:pk>/', views.gallery_view, name="index"),
]

if settings.DEBUG:
        urlpatterns += static(settings.MEDIA_URL,
                              document_root=settings.MEDIA_ROOT)


import argparse
from douyin_image.converter import add_effect

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--image", help="path of image to add effect", dest="image")
    parser.add_argument("--out", help="path of output image", dest="out")
    args = parser.parse_args()

    if args.image is None:
        parser.error("please specify or --image")

    out = args.out if args.out is not None else "out.jpeg"
    add_effect(args.image, out)


#! /usr/bin/python

import pyglet

from pyglet.window import key, NoSuchConfigException

def onKeyPressHandler(p_key, p_mods):
    """
    :param p_key:
    :param p_mods:
    """
    format_str = 'Key pressed: "{}" mods: {}'
    char = key.symbol_string(p_key)
    label.text = format_str.format(char, p_mods)

def onDrawHandler():
    window.clear()
    label.x = window.width // 2
    label.y = 0 + label.content_height
    label.draw()

    image.blit(window.width // 2, window.height // 2)

# This describes the system we;re running on
platform = pyglet.window.get_platform()

# This gets the default display device (e.g.: graphics card)
display = platform.get_default_display()

screen = display.get_default_screen()
# Check if we can find a right most screen
for available_screen in display.get_screens():
    if available_screen.x != 0:
        screen = available_screen

try:
    # try to employ multisampling
    template = pyglet.gl.Config(
        sample_buffers=1,
        samples=4,
        alpha_size=8
    )
    config = screen.get_best_config(template)
    print("Using multisampling")
except NoSuchConfigException:
    template = pyglet.gl.Config()
    config = screen.get_best_config(template)
    print("resort to best default config")

context = config.create_context(None)

window = pyglet.window.Window(
    context=context,
    fullscreen=True
)
# window.push_handlers(pyglet.window.event.WindowEventLogger())
window.on_key_press = onKeyPressHandler
window.on_draw = onDrawHandler

image = pyglet.resource.image('StN_logo.jpg')

label = pyglet.text.Label(
    text='Press a key',
    font_name='Times New Roman',
    font_size=36,
    anchor_x='center',
    anchor_y='top'
)

pyglet.app.run()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import sys, os

from sqlalchemy import create_engine
from sqlalchemy.orm import *
sys.path.append(os.path.abspath("/root/ArturAuto"))
from lib import UTM, ORM

engine = create_engine("mysql+pymysql://progress:progress@10.8.16.18:3306/progress?charset=utf8mb4")
session = Session(bind=engine)

def main(port):
    utm = UTM.UTM(port=port)
    ttn_NATTN = utm.get_ReplyNATTN()
    accepted_ttn = utm.get_accepted_ttn()

    if not ttn_NATTN:
        print("\033[0;32m{}\033[0m".format("Нету накладных в ReplyNATTN"))
        sys.exit()

    print("\033[1;33mВыберите накладные\033[0m")
    print("\033[0;35m1: \033[0;33mПринять все накладные\033[0m")
    for index, ttn in enumerate(ttn_NATTN):
        index += 2
        shipper = session.query(ORM.Shippers).filter(ORM.Shippers.fsrar == ttn[3]).first()
        if ttn[0] in accepted_ttn:
            fullname_ttn = "\033[0;35m{0}: \033[1;36m{1} \033[1;37m{2} \033[1;36m{3} \033[1;37m{4} \033[1;33m{5}\033[0m". \
                            format(str(index), shipper.name, str(ttn[1]), str(ttn[2]), str(ttn[0]), "Уже принята")
        else:
            fullname_ttn = "\033[0;35m{0}: \033[1;36m{1} \033[1;37m{2} \033[1;36m{3} \033[1;37m{4}\033[0m". \
                            format(str(index), shipper.name, str(ttn[1]), str(ttn[2]), str(ttn[0]))
        print(fullname_ttn)
    answer = int(input("Введите строку: "))
    idnex_answer = answer - 2
    if answer == 1:
        for ttn in ttn_NATTN:
            utm.send_WayBillv4(ttn[0])
    elif answer == 0:
        print("\033[0;31m{}\033[0m".format("Недопустимое число 0"))
    elif answer > len(ttn_NATTN) + 1:
        print("\033[0;31m{0} {1}\033[0m".format("Недопустимое число", str(answer)))
    else:
        answer_ttn = ttn_NATTN[idnex_answer]
        ttn = answer_ttn[0]
        utm.send_WayBillv4(ttn)

if len(sys.argv) == 2:
    main(sys.argv[1])
else:
    print("Укажите один аргумент в виде порта УТМ")


from django.urls import path, include
from rest_framework.routers import DefaultRouter

from client import views

router = DefaultRouter()
router.register('company', views.CompanyViewSet)        # Company API URL - /api/client/company/
router.register('logs', views.LogViewSet)                # Log API URL     - /api/client/logs/
router.register('clients', views.ClientViewSet)          # Client API URL  - /api/client/clients/

app_name = 'client'

urlpatterns = [
    path('', include(router.urls))
]


'''
There is an undirected graph with n nodes numbered from 0 to n - 1 (inclusive). You are given a 0-indexed integer array values where values[i] is the value of the ith node. You are also given a 0-indexed 2D integer array edges, where each edges[j] = [uj, vj, timej] indicates that there is an undirected edge between the nodes uj and vj, and it takes timej seconds to travel between the two nodes. Finally, you are given an integer maxTime.

A valid path in the graph is any path that starts at node 0, ends at node 0, and takes at most maxTime seconds to complete. You may visit the same node multiple times. The quality of a valid path is the sum of the values of the unique nodes visited in the path (each node's value is added at most once to the sum).

Return the maximum quality of a valid path.

Note: There are at most four edges connected to each node.

Example 1:

Input: values = [0,32,10,43], edges = [[0,1,10],[1,2,15],[0,3,10]], maxTime = 49
Output: 75
Explanation:
One possible path is 0 -> 1 -> 0 -> 3 -> 0. The total time taken is 10 + 10 + 10 + 10 = 40 <= 49.
The nodes visited are 0, 1, and 3, giving a maximal path quality of 0 + 32 + 43 = 75.
Example 2:

Input: values = [5,10,15,20], edges = [[0,1,10],[1,2,10],[0,3,10]], maxTime = 30
Output: 25
Explanation:
One possible path is 0 -> 3 -> 0. The total time taken is 10 + 10 = 20 <= 30.
The nodes visited are 0 and 3, giving a maximal path quality of 5 + 20 = 25.
Example 3:

Input: values = [1,2,3,4], edges = [[0,1,10],[1,2,11],[2,3,12],[1,3,13]], maxTime = 50
Output: 7
Explanation:
One possible path is 0 -> 1 -> 3 -> 1 -> 0. The total time taken is 10 + 13 + 13 + 10 = 46 <= 50.
The nodes visited are 0, 1, and 3, giving a maximal path quality of 1 + 2 + 4 = 7.
Example 4:

Input: values = [0,1,2], edges = [[1,2,10]], maxTime = 10
Output: 0
Explanation:
The only path is 0. The total time taken is 0.
The only node visited is 0, giving a maximal path quality of 0.

Constraints:

n == values.length
1 <= n <= 1000
0 <= values[i] <= 108
0 <= edges.length <= 2000
edges[j].length == 3
0 <= uj < vj <= n - 1
10 <= timej, maxTime <= 100
All the pairs [uj, vj] are unique.
There are at most four edges connected to each node.
The graph may not be connected.
'''
from collections import defaultdict
class Solution:
    def maximalPathQuality(self, values, edges, maxTime: int) -> int:
        adj_matrix = defaultdict(list)
        for a, b, t in edges:
            adj_matrix[a].append((b, t))
            adj_matrix[b].append((a, t))
        # print(adj_matrix)
        # node, timeLeft, nodes_visited, path_used
        q = [(0, 0, {0}, set())]
        res = values[0]
        while len(q) > 0:
            node, timeUsed, nodes_visited, path_used = q.pop(0)
            print(node, timeUsed, nodes_visited, path_used)
            for next_node, time_cost in adj_matrix[node]:
                if timeUsed+time_cost <= maxTime:
                    path = (min(node, next_node), max(node, next_node))
                    new_nodes_visited = set.union(nodes_visited, {next_node})
                    if next_node == 0 or 2*(timeUsed + time_cost) <= maxTime:
                        value = sum([values[i] for i in list(new_nodes_visited)])
                        res = max(value, res)
                    if path not in path_used:
                        new_path_used= set.union(path_used, {path})
                        q.append((next_node, timeUsed+time_cost, new_nodes_visited, new_path_used))
                    if 2*(timeUsed + time_cost) <= maxTime:
                        new_path_used= set.union(path_used, {path})
                        q.append((0, 2*(timeUsed+time_cost), new_nodes_visited, new_path_used))
        return res

s = Solution()
values = [0,32,10,43]
edges = [[0,1,10],[1,2,15],[0,3,10]]
maxTime = 49
res = s.maximalPathQuality(values, edges, maxTime)
print(res)


#!/usr/bin/env python3

import os, sys
import datetime

import config
config.datasets_data = config.get_datasets_data_analysis()
config.datasets_mc = config.get_datasets_mc_analysis()

import time

import getpass
username = getpass.getuser()

import subprocess

import hashlib

import yaml
config_doc = yaml.load(open("config/config.yaml", "r"), Loader=yaml.FullLoader)

import argparse
parser = argparse.ArgumentParser()
parser.add_argument("-sub", "--submit", help="Submit jobs", action="store_true")
parser.add_argument("-mon", "--monitor", help="Monitor tasks", action="store_true")
parser.add_argument("-cle", "--clear", help="Clear workspace", action="store_true")
parser.add_argument("-d", "--dataset", help="Dataset to be analyzed. If not specified, all datasets will be submited.")
parser.add_argument("-y", "--year", help="Year for MC dataset. To be used only with --dataset (-d) option")
parser.add_argument("-c", "--check", help="Check jobs logs and print errors.", action="store_true")
# args = parser.parse_args()
args, unknown = parser.parse_known_args()

##############
# Celery App #
##############
from celery import Celery

app = Celery('pbs_ggntuples_analyzer', backend=f'redis://{config_doc["celery_master_node_ip"]}:{config_doc["redis_port"]}/1', broker=f'redis://{config_doc["celery_master_node_ip"]}:{config_doc["redis_port"]}/0')

app.conf.update(
    CELERY_TASK_RESULT_EXPIRES=0,
)

@app.task
def run_taks(ds):
  print("\n\n--> Starting H/Z --> Upsilon + Photon analyzer for: " + ds["dataset_name"] + " - "+ ds["year"]+" - Chunk: "+ds["chunk"])
  exec_command = f'./ggntuples_analyzer.py -d {ds["dataset_name"]} -y {ds["year"]} -c {ds["chunk"]} > pbs_buffer/logs/log_{ds["dataset_name"]}_{ds["year"]}_{ds["chunk"]}.log 2>&1 '
  exec_command = f'./ggntuples_analyzer.py -d {ds["dataset_name"]} -y {ds["year"]} -c {ds["chunk"]} > pbs_buffer/logs/log_{ds["dataset_name"]}_{ds["year"]}_{ds["chunk"]}.log 2>&1 '
  proc_child = subprocess.Popen(exec_command, stdout=subprocess.PIPE, shell=True)
  proc_child.communicate()[0]
  return proc_child.returncode

def do_submit(ds):
  # run_taks.delay(ds)
  run_taks.apply_async(args=(ds,), kwargs=None, task_id=ds["dataset_name"]+'_'+ds["year"]+'_'+ds["chunk"])

def do_monitor():
  try:
    print("\n\n--> Setup tunnel:")
    print(r"ssh -N -f -L 5556:localhost:5556 ftaraujo@planck")
    print("Access: http://localhost:5556\n\n")
    print("--> Press Ctrl+C to exit.")
    os.system("flower -A pbs_ggntuples_analyzer --port=5556")

  except KeyboardInterrupt:
      print ('--> Exiting...')

def start_redis():
  os.system("redis-server --bind 0.0.0.0 --port 4023 > pbs_buffer/redis.log 2>&1  &")

def clear_workspace():
  # os.system("rm -rf pbs_buffer/*")
  os.system(f"qselect -u {username} | xargs qdel")
  os.system("pkill -9 -f celery*")
  os.system("pkill -9 -f flower*")
  os.system("pkill -9 -f redis-server*")

pbs_sub_file = r"""
#!/bin/bash

# send email
#PBS -M ftaraujo@ifi.unicamp.br
#PBS -m abe

### Process name
#PBS -N pbs_ggntuples_analyzer___SUB_NUMBER__

### Output files
#PBS -e pbs_buffer/worker_log___SUB_NUMBER___err.log
#PBS -o pbs_buffer/worker_log___SUB_NUMBER___out.log

# queues
#PBS -q __QUEUE__
#PBS -l nodes=__NODES__:ppn=__PPN__

# enter working dir
cd $PBS_O_WORKDIR
echo "Location:"
pwd
# ls -d /cvmfs/cms.cern.ch/cmsset_default.sh

# load env
source ~/.bashrc
source /cvmfs/cms.cern.ch/cmsset_default.sh
conda deactivate
conda deactivate
conda activate HZUpsilonPhotonRun2Env
eval `scramv1 runtime -sh`
export PYTHONPATH=$CONDA_PREFIX/lib/python3.6/site-packages/
source /cvmfs/sft.cern.ch/lcg/releases/LCG_92python3/ROOT/6.12.04/x86_64-centos7-gcc7-opt/bin/thisroot.sh # for bash

# start worker
celery -A pbs_ggntuples_analyzer worker --loglevel=info --concurrency=__CONC__ --hostname=worker___SUB_NUMBER__@%h___NAME_APPEND__

exit
"""

def launch_workers(n_jobs):
  print("\n\n--> To launch workers...")
  for w in range(n_jobs):
    with open(f"pbs_buffer/pbs_sub_file_{w}.sh", "w") as sub_file:
      pbs_sub_file_buff = pbs_sub_file.replace("__SUB_NUMBER__", str(w)).replace("__QUEUE__", config_doc["queue"]).replace("__PPN__", str(config_doc["ppn"])).replace("__NODES__", str(config_doc["nodes"])).replace("__CONC__", str(config_doc["concurrency"]))
      pbs_sub_file_buff = pbs_sub_file_buff.replace("__NAME_APPEND__", str(hashlib.sha1(pbs_sub_file.encode("UTF-8")).hexdigest()[:10]))
      sub_file.write(pbs_sub_file_buff)
      os.system(f"chmod +x pbs_buffer/pbs_sub_file_{w}.sh")
      # os.system(f"qstat -q")
      # os.system(f"qsub -q {config_doc['queue']} pbs_buffer/pbs_sub_file_{w}.sh")
      print(f"qsub -q {config_doc['queue']} pbs_buffer/pbs_sub_file_{w}.sh")
  print("\n\n")

if __name__ == '__main__':
  start_time = time.time()

  if args.submit:
    print("\n\n--> Starting ---")
    os.system("rm -rf pbs_buffer/*")
    os.system("mkdir -p pbs_buffer/logs")

    ############################
    # Redis Server
    ############################
    print("--> Launching Redis Server...")
    start_redis()

    ############################
    # launch workers
    ############################
    launch_workers(config_doc["n_jobs"])

    ############################
    # dataset selector
    ############################
    print("--> Filtering selected datasets...")
    # check if a specific dataset has been requested
    if args.dataset != None:
      if args.year == None and not(args.dataset.startswith("Run")):
        print("\n\nERROR: Dataset and Year should be set together!")
        exit()
      args.background = True
      isData = args.dataset.startswith("Run")
      if isData:
        _buff = []
        for d in config.datasets_data:
          if d["dataset_name"] == args.dataset:
            _buff.append(d)
        if _buff == []:
          print("\n\nDataset + chunk not found!")
          exit()
        config.datasets_data = _buff
        config.datasets_mc = []
      else:
        _buff = []
        for d in config.datasets_mc:
          if d["dataset_name"] == args.dataset and d["year"] == args.year:
            _buff.append(d)
        if _buff == []:
          print("\n\nDataset + year + chunk not found!")
          exit()
        config.datasets_data = []
        config.datasets_mc = _buff

    #############################
    # Submit for Data
    #############################
    print("--> Submiting tasks - Data...")
    print(len(config.datasets_data))
    for ds in config.datasets_data:
      do_submit(ds)

    #############################
    # Submit for MC
    #############################
    print("--> Submiting tasks - MC...")
    for ds in config.datasets_mc:
      do_submit(ds)

    print("\n\n--> Done in %s seconds ---" % datetime.timedelta(seconds=(time.time() - start_time)))
    exit()

  if args.monitor:
    #############################
    # Monitoring
    #############################
    print("--> Starting Flower...")
    do_monitor()

  if args.clear:
    #############################
    # Clear Workspace
    #############################
    print("--> Clearing workspace...")
    clear_workspace()

  if (args.submit == False) and (args.monitor == False) and (args.clear == False):
    print("Try execute it with --help.")


import logging

import os
import torch
import contextlib
import numpy as np
import SimpleITK as sitk
import random
from pathlib import Path

def check_dir(dir_path, create=False):
    '''
    check the existence of a dir, when create is True, will create the dir if it does not exist.
    dir_path: str.
    create: bool
    return:
    exists (1) or not (-1)
    '''
    if os.path.exists(dir_path):
        return 1
    else:
        if create:
            os.makedirs(dir_path)
        return -1

def load_image_label(image_path, label_path=None, slice_id=0, crop_size=(192, 192)):
    """[summary]
    loads image and labels (optional) from disk (Nifti, nrrd)
    return cropped 3dim numpy array  [DHW]
    Args:
        image_path ([type]): [description]
        label_path ([type]): [description]
        slice_id (int, optional): [description]. Defaults to 0.
        crop_size (tuple, optional): [description]. Defaults to (192, 192).
    Returns:
        [type]: [description]
    """
    support_formats = ['.nrrd', '.nii', '.nii.gz']
    assert Path(image_path).suffix in support_formats, 'only support loading images/labels with extensions:{}.'.format(
        str(support_formats))

    image = sitk.GetArrayFromImage(sitk.ReadImage(image_path))[slice_id]
    # print('image size:', image.shape)

    h_diff = (image.shape[0]-crop_size[0])//2
    w_diff = (image.shape[1]-crop_size[1])//2

    cropped_image = image[h_diff:crop_size[0] +
                          h_diff, w_diff:crop_size[1]+w_diff]
    # rescale image intensities to 0-1
    cropped_image = (cropped_image-cropped_image.min()) / \
        (cropped_image.max()-cropped_image.min()+1e-10)

    if label_path is not None:
        label = sitk.GetArrayFromImage(sitk.ReadImage(label_path))[slice_id]
        # logging.info('label size:', label.shape)
        assert image.shape == label.shape, "The sizes of the input image and label do not match, image:{}label:{}".format(
            str(image.shape), str(label.shape))
        cropped_label = label[h_diff:crop_size[0] +
                              h_diff, w_diff:crop_size[1]+w_diff]

        return cropped_image, cropped_label
    else:
        return cropped_image

def rescale_intensity(data, new_min=0, new_max=1, eps=1e-20):
    '''
    rescale pytorch batch data
    :param data: N*1*H*W
    :return: data with intensity ranging from 0 to 1
    '''
    bs, c, h, w = data.size(0), data.size(1), data.size(2), data.size(3)
    data = data.view(bs*c, -1)
    old_max = torch.max(data, dim=1, keepdim=True).values
    old_min = torch.min(data, dim=1, keepdim=True).values
    new_data = (data - old_min) / (old_max - old_min + eps) * \
        (new_max-new_min)+new_min
    new_data = new_data.view(bs, c, h, w)
    return new_data

def check_dir(dir_path, create=False):
    '''
    check the existence of a dir, when create is True, will create the dir if it does not exist.
    dir_path: str.
    create: bool
    return:
    exists (1) or not (-1)
    '''
    if os.path.exists(dir_path):
        return 1
    else:
        if create:
            os.makedirs(dir_path)
        return -1

@contextlib.contextmanager
def _disable_tracking_bn_stats(model):
    def switch_attr(model, new_state=None, hist_states=None):
        """[summary]

        Args:
            model ([torch.nn.Module]): [description]
            new_state ([bool], optional): [description]. Defaults to None.
            hist_states ([type], optional): [description]. Defaults to None.

        Returns:
            [type]: [description]
        """
        old_states = {}
        for name, module in model.named_children():
            if hasattr(module, 'track_running_stats'):
                old_state = module.track_running_stats
                if hist_states is not None:
                    module.track_running_stats = hist_states[name]
                else:
                    if new_state is not None:
                        module.track_running_stats = new_state
                old_states[name] = old_state
        return old_states

    old_states = switch_attr(model, new_state=False)
    yield
    switch_attr(model, hist_states=old_states)

def set_grad(module, requires_grad=False):
    for p in module.parameters():  # reset requires_grad
        p.requires_grad = requires_grad

def random_chain(alist, *args):
    """[select a sub list from  a list and chained them in a random order]

    Args:
        list ([type]): [description]
    """
    length = len(alist)
    assert length >= 1, "input list must contains at least one element"
    if length == 1:
        results = [alist]
        if len(args) > 0:
            for arg in args:
                assert len(arg) == 1, 'must share equal size'
                results.append(arg)

            return results
        else:
            return results[0]

    sub_len = np.random.randint(low=1, high=length+1)

    r = random.random()            # randomly generating a real in [0,1)
    # lambda : r is an unary function which returns r
    random.shuffle(alist, lambda: r)
    # using the same function as used in prev line so that shuffling order is the same
    results = []
    results.append(alist[:sub_len])
    if len(args) >= 0:
        for arg in args:
            random.shuffle(arg, lambda: r)
            results.append(arg[:sub_len])

    return results


from collections import namedtuple
import mimetypes
import urlparse
from magic import Magic
import zope.interface
import zope.component
from zope.component.hooks import getSite
from plone.registry.interfaces import IRegistry

from pmr2.app.workspace.exceptions import *
from pmr2.app.workspace.interfaces import IStorage, IStorageUtility
from pmr2.app.workspace.interfaces import IWorkspace

ProtocolResult = namedtuple('ProtocolResult', ('result', 'event'))

class BaseStorage(object):
    """\
    Basic storage type
    """

    zope.interface.implements(IStorage)

    def __init__(self, context):
        # context will always be the immediate object that leads to the
        # creation of this storage object, or the final object that is
        # resolved.
        #
        # example 1: if we have an exposure of a workspace, and we adapt
        # an exposure to a storage, its context will be the workspace
        # and not the exposure
        #
        # example 2: if we have an exposure file that sits within an
        # exposure, context will again be the workspace that the
        # exposure references.
        #
        # to get to an individual file, just specify the path.

        self.context = context

    # Parameters

    @property
    def rev(self):
        raise NotImplementedError

    @property
    def shortrev(self):
        return self.rev

    # Datetime implementation

    # We assume UTC for all datetime.

    __default_datefmt = {
        'rfc2822': '%a, %d %b %Y %H:%M:%S +0000',
        'rfc3339': '%Y-%m-%dT%H:%M:%SZ',
        'iso8601': '%Y-%m-%d %H:%M:%S',
    }

    __datefmt = 'iso8601'

    # this stores the types of archive available, with its key the
    # method name with a 3-tuple value that stores the human readable
    # name, file name extension and mimetype.  See tests/storage.py for
    # the example.
    _archiveFormats = {}
    __archiveFormatInfoKeys = ('name', 'ext', 'mimetype')

    def _getDatefmt(self):
        return self.__datefmt

    def _setDatefmt(self, datefmt):
        if datefmt in BaseStorage.__default_datefmt:
            self.__datefmt = datefmt
        else:
            raise ValueError('unsupported datetime format')

    datefmt = property(_getDatefmt, _setDatefmt)

    @property
    def datefmtstr(self):
        return BaseStorage.__default_datefmt[self.datefmt]

    @property
    def builtin_archive_formats(self):
        # should return a list of builtins available, could be
        # automatically generated.
        archiveFormats = {}
        return archiveFormats

    @property
    def archiveFormats(self):
        return sorted(self._archiveFormats.keys() +
            self.builtin_archive_formats.keys())

    # Navigational list

    _lastnav = None

    @property
    def lastnav(self):
        if self._lastnav is None:
            return []
        else:
            return self._lastnav

    # Default implementation for methods

    def archiveInfo(self, format):
        info = self._archiveFormats.get(format,
            self.builtin_archive_formats.get(format, None))
        if not info:
            raise ValueError('format `%s` not supported' % format)
        return dict(zip(self.__archiveFormatInfoKeys, info))

    def archive(self, format):
        archivePrefix = 'archive_'
        if format not in self.archiveFormats:
            raise ValueError('format `%s` not supported' % format)
        archiver = getattr(self, archivePrefix + format, None)
        if not (archiver and callable(archiver)):
            # Well someone screwed up by forgetting to implement the
            # appropriate method with the correct name.
            raise NotImplementedError
        return archiver()

    def basename(self, name):
        raise NotImplementedError

    def checkout(self, rev=None):
        raise NotImplementedError

    def file(self, path):
        raise NotImplementedError

    def fileinfo(self, path):
        raise NotImplementedError

    def files(self):
        raise NotImplementedError

    def format(self, permissions, node, date, size, path, contents,
               contenttype=None, author='', desc='', external=None, *a, **kw):

        # need a way to derive the correct baseview.
        baseview = 'file'

        def mimetype():
            # use the built-in mimetypes, then use magic library
            # XXX performance penalty here getting the entire file?
            # XXX should provide a way to allow other packages to add
            # more mimetypes.
            data = contents()
            if not isinstance(data, basestring):
                return None
            mt = mimetypes.guess_type(path)[0]
            if mt is None or mt.startswith('text/'):
                magic = Magic(mime=True)
                mt = magic.from_buffer(data[:4096])
            return mt

        return {
            'permissions': permissions,
            'author': author,
            'desc': desc,
            'node': node,
            'date': date,
            'size': size,
            'baseview': baseview,
            'file': path,
            'fullpath': None,
            'contenttype': contenttype,
            'basename': self.basename(path),
            'contents': contents,
            'mimetype': mimetype,
            'external': external,
        }

    def listdir(self, path):
        raise NotImplementedError

    def log(self, start, count, branch=None, *a, **kw):
        raise NotImplementedError

    def pathinfo(self, path):
        """\
        This is a default implementation.  If a specific backend
        supports other types of path resolution, this method can be
        safely overridden provided the correct data is returned.
        """

        try:
            listing = self.listdir(path)
            # consider using an iterator?
            contents = lambda: listing
            data = self.format(**{
                'permissions': 'drwxr-xr-x',
                'node': self.rev,
                'date': '',
                'size': '',
                'path': path,
                'contents': contents,
                'author': '',
                'desc': '',
            })
        except PathNotDirError:
            # then path must be a file
            data = self.fileinfo(path)
        return data

    def roots(self, rev=None):
        raise NotImplementedError

    def clonecmd(self):
        # Default to an empty string to denote a default; the original
        # implementation only has this visible via the portlet, so this
        # tells it to use the default handler.
        return ''

    def resolve_file(self, path):
        """
        This default implementation relies on the underlying file() and
        pathinfo() method in working order, and that the relevant
        registry settings to resolve hostnames for local subrepos be
        available, as this method will not attempt to resolve or fetch
        any external resources hosted outside of this instance.
        """

        def external_file(location, rev, subpath):
            p = urlparse.urlparse(location)
            if not p.netloc in mappings:
                raise SubrepoPathUnsupportedError(
                    'requested path at %r requires subrepo at unsupported '
                    'netloc %r when trying to resolve subpath %r' % (
                        path, p.netloc, subpath))

            objpath = mappings[p.netloc] + p.path
            workspace = portal.restrictedTraverse(objpath.split('/'))
            storage = IStorage(workspace)
            return file(storage, subpath)

        def file(storage, subpath):
            pathinfo = storage.pathinfo(subpath)
            if pathinfo.get('external'):
                external = pathinfo['external']
                return external_file(
                    external['location'], external['rev'], external['path'])
            # Using this instead of contents to ignore directories.
            return storage.file(subpath)

        registry = zope.component.queryUtility(IRegistry)
        if not registry:
            # no registry, so no resolution
            return self.file(path)

        mappings = registry.get('pmr2.app.settings.prefix_maps') or {}
        portal = getSite()
        return file(self, path)

class StorageUtility(object):
    """\
    Basic storage utility.

    Reasons why we have this instead of straight up adapter follows:

    Usage in vocabularies with a context.  The given context may be of a
    type that cannot be adapted into a Storage object.  As the storage
    types are provided globally, it makes more sense to make this a
    globally available utility.  While the subclass of BaseStorage can
    be used instead, this serves as a more lightweight utility rather
    than registering the implementation of the whole storage class.

    Second, this allows the other classes to subclass Storage without
    worrying about including this for cases where independence from the
    Workspace concept is desired.

    Third, initialization for the base storage class is now independent
    of implementation of the utility.
    """

    zope.interface.implements(IStorageUtility)

    def create(self, context):
        raise NotImplementedError

    def acquireFrom(self, context):
        raise NotImplementedError

    def isprotocol(self, request):
        raise NotImplementedError

    def validateExternalURI(self, uri):
        # hardcoding these as default valid for now.
        validURI = ['http://', 'https://', ]
        for i in validURI:
            if uri.startswith(i):
                return True
        return False

    def protocol(self, context, request):
        raise NotImplementedError

    def syncWorkspace(self, context, workspace):
        raise NotImplementedError

    def syncIdentifier(self, context, identifier):
        raise NotImplementedError

    def sync(self, context, source):
        if IWorkspace.providedBy(source):
            return self.syncWorkspace(context, source)
        return self.syncIdentifier(context, source)

    def __call__(self, context):
        return self.acquireFrom(context)


import cv2
import numpy as np
from imutils import paths
import os
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.preprocessing import LabelEncoder,scale
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report,confusion_matrix #查看预测结果
from skimage import feature as ft
from sklearn.cross_validation import cross_val_score
from sklearn.feature_selection import SelectKBest,chi2 #卡方检验
from sklearn.decomposition import PCA
import pickle #导入和生成模型
import datetime

startime = datetime.datetime.now()

#导入图像数据
#imgPaths = list(paths.list_images("F:\pycharm_project\First_image_classifier\leedsbutterfly\images"))
imgPaths = list(paths.list_images("leedsbutterfly/data_bf"))
# imgPaths = list(paths.list_images("train_4000"))
data = []
labels = []

for (i, imgPath) in enumerate(imgPaths):
    img = cv2.imread(imgPath)
    img = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
    #np.array.flatten()将数组变为一维
    img = cv2.resize(img,(32,32))
    feature,hogimage= ft.hog(img, orientations=9, pixels_per_cell=(6, 6),
                                cells_per_block=(3, 3), transform_sqrt=None, visualise=True)
    #这里imgPath.split(os.path.sep)[-1]是子目录下的文件名.jpg，而[-2]是子目录的名字,从而区分标签
    label = imgPath.split(os.path.sep)[-2]
    data.append(feature)
    labels.append(label)
    print("Image Loading ....")
    print("Processed {}/{}".format(i+1,len(imgPaths)))

data1 = np.array(data)
labels1 = np.array(labels)

#将labels1 转成整形表示
le = LabelEncoder()
labels2 = le.fit_transform(labels1)
#用卡方检验选择 与目标相关性高的特征 KNN 的k=380 ，精度在0.64，SVM的k=500 ，精度在0.63
data2 = SelectKBest(chi2, k=380).fit_transform(data1, labels2) #这里的data1必须是非负矩阵，所以需要先用卡方检验在标准化
#对特征选择之后想通过PCA无监督降维的方式提高效率，但是发现精度也相应减少，由原本的0.63减少到0.58
# data2 = PCA(n_components=100).fit_transform(data2, labels2)
#将每一列特征进行标准化正太分布（均值为0，方差为1）
data2 = scale(data2) #这里的data2是含负数的矩阵

#将数据集进行分类
(trainX,testX,trainY,testY) = train_test_split(data2,labels2,test_size=0.25,random_state=42)
model = SVC(C=10, kernel = 'rbf', gamma = 'auto',class_weight = 'balanced') #SVM中卡方检验用的K = 500
# model =KNeighborsClassifier(p=1, n_neighbors=92, weights='uniform', algorithm='auto')
score = cross_val_score(model,trainX,trainY,cv = 10,scoring='accuracy')
print(score.mean())
# model =KNeighborsClassifier(p=1, n_neighbors=92, weights='uniform', algorithm='auto')
# model = SVC(C=10, kernel = 'rbf', gamma = 'auto',class_weight = 'balanced',probability=True)
model.fit(trainX,trainY)
#保存模型
# pickle.dump(model,open('model_save/model.model','wb'))
#导入模型
# model = pickle.load(open('model_save/model.model','wb'))
print(classification_report(testY,model.predict(testX)))

#看混淆矩阵的pt值
pt = confusion_matrix(testY,model.predict(testX))
print(pt)

#随机挑选1个样本进行测试
instance = testX[100:109]
#print(instance)
a = instance[0] #这里的a值是列的形式，而model.predict的单个输入必须是行的形式，因此要reshape
test1 = model.predict(a.reshape(1,-1))
instance_label = testY[100:109]
true1 = instance_label[0]
print('instance[0] prediction class is {}'.format(test1))
print('instance[0] true class is {}'.format(true1))

endtime = datetime.datetime.now()
print("Using Time Is {}".format(endtime - startime))


import torch as t
import torchvision as tv
from torchvision import transforms
from PIL import Image
import json
import numpy as np
class wavelet_GPU():
    '''
    Used to stored the mateix of wavelet
    '''
    def __init__(self):
        self.config = json.load(open('config.json'))
        self.DEVICE = self.config["DEVICE"]
        self.M_dict = {}
        self.iM_dict = {}

    def dwt(self, data):
        size = data.shape[-1]
        idx = (int)(np.log2(size))
        try:
            # TO test if have the weight stored in the dict
            self.M_dict['{}_{}'.format(size, idx)]
            #print('Exist In the Dict')
        except KeyError:
            loop = idx
            #print('Not Exist In the Dict')
            self.M_dict['{}_{}'.format(size, idx)] = []
            while(loop != 0):
                w = t.eye(size, dtype=t.float, device=self.DEVICE)
                w[:2*loop, :2*loop] = 0
                for i in range(loop):
                    w[2*i, i] = 0.5
                    w[2*i+1, i] = 0.5
                    w[2*i, i+loop] = 0.5
                    w[2*i+1, i+loop] = -0.5
                self.M_dict['{}_{}'.format(size, idx)].append(w)
                #print(w)
                loop -= 1
        finally:
            for w in self.M_dict['{}_{}'.format(size, idx)]:
                data = t.matmul(data, w)
            #print(data)
            return data

    def idwt(self, data):
        size = data.shape[-1]
        idx = (int)(np.log2(size))
        try:
            # TO test if have the weight stored in the dict
            self.iM_dict['{}_{}'.format(size, idx)]
            #print('Exist In the Dict')
        except KeyError:
            loop = idx
            #print('Not Exist In the Dict')
            try:
                self.iM_dict['{}_{}'.format(size, idx)] = [t.inv(item) for item in self.M_dict['{}_{}'.format(size, idx)]]
                self.iM_dict['{}_{}'.format(size, idx)].reverse()
            except:
                self.iM_dict['{}_{}'.format(size, idx)] = []
                while(loop != 0):
                    w = t.eye(size, dtype=t.float,device=self.DEVICE)
                    w[:2*loop, :2*loop] = 0
                    for i in range(loop):
                        w[2*i, i] = 0.5
                        w[2*i+1, i] = 0.5
                        w[2*i, i+loop] = 0.5
                        w[2*i+1, i+loop] = -0.5
                    self.iM_dict['{}_{}'.format(size, idx)].append(t.inv(w))
                    #print(w)
                    loop -= 1
                self.iM_dict['{}_{}'.format(size, idx)].reverse()
        finally:
            for w in self.iM_dict['{}_{}'.format(size, idx)]:
                data = t.matmul(data, w)
            #print(data)
            return data

if __name__ == "__main__":
    a = t.tensor([[9, 7, 3, 5]], dtype=t.float).to('cuda')
    wavelet = wavelet_GPU()
    out = wavelet.dwt(a)
    out = wavelet.idwt(out)


import scipy.constants as sc

#***** Pressure broadening settings *****#

gamma_0_fixed = 0.07    # If fixed broadening chosen, use this Lorentzian HWHM
n_L_fixed = 0.50        # If fixed broadening chosen, use this temperature exponent

T_ref = 296.0   # Reference temperature for broadening parameters
P_ref = 1.0     # Reference temperature for EXOMOL broadening parameters (bar) - HITRAN conversion from atm already pre-handled

#***** Define physical constants *****#

c = sc.c          # Speed of light (SI) = 299792458.0 m s^-1
kb = sc.k         # Boltzmann constant (SI) = 1.38064852e-23 m^2 kg s^-2 K^-1
h = sc.h          # Planck's constant (SI) = 6.62607004e-34 m^2 kg s^-1
m_e = sc.m_e      # Electron mass (SIT) = 9.10938356e-31 kg
c2 = h*c/kb       # Second radiation constant (SI) = 0.0143877736 m K
c2 *= 100.0       # Convert to cm K for intensity formulae
u = sc.u          # Unified atomic mass unit (SI) = 1.66053904e-27 kg
pi = sc.pi        # pi = 3.141592653589793
Ryd = sc.Rydberg  # Rydberg constant


from enum import Enum

class LabelType(Enum):
    """Тип элемента в области видимости."""
    FUNC = "function"
    VAR = "variable"

class Label:
    """Класс, описывающий элемент области видимости."""
    def __init__(self, label_type: LabelType, name, val):
        self.label_type = label_type
        self.name = name
        self.val = val

class Scope:
    """Класс, описывающий область видимости."""
    def __init__(self, prev_scope):
        self._labels = []
        self.children_scopes = []
        self.prev_scope = prev_scope

    def add_label(self, label: Label, loc):
        """Этот метод добавляет функцию или переменную."""
        label_names = [lbl.name for lbl in self._labels]
        # Если эл-т не существует в области видимости, мы его добавляем в неё
        if label.name not in label_names:
            self._labels.append(label)
            return True
        else:
            # В противном случае мы проверяем, не существует ли функции с таким же именем.
            index = label_names.index(label)
            if label.label_type is LabelType.FUNC or self._labels[index].label_type is LabelType.FUNC:
                # Если она существует, выводим ошибку.
                raise SemanticException("Объект с именем {} уже существует.".format(label.name), loc[0], loc[1])
            else:
                # Если существует переменная с таким же именем, мы переопределяем её значение.
                self._labels[index] = label

    def get_label(self, name, loc):
        """Метод, возвращающий переменную из текущей области видимости или из её родительской области видимости."""
        result = [lbl for lbl in self._labels if lbl.name == name]
        if len(result) > 0:
            # Если переменная или функция с таким именем существует, возвращаем её.
            return result[0]
        else:
            if self.prev_scope is not None:
                # Если она не существует в текущей области видимости, ищем её в родительской области видимости.
                return self.prev_scope.get_label(name, loc)
            else:
                # Если такой переменной или функции нет, то выводим ошибку.
                raise SemanticException("Объекта с именем {} не существует. ".format(name), loc[0], loc[1])

class SemanticException(Exception):
    """Исключения в семантическом анализе."""
    def __init__(self, message, row: int = None, col: int = None) -> None:
        if row or col:
            message += " ("
            if row:
                message += "строка: {}".format(row)
                if col:
                    message += ", "
            if row:
                message += "символ: {}".format(col)
            message += ")"
        self.message = message


import logging

from pynamodb.exceptions import UpdateError

from common.error_inspector import ErrorInspector
from common.measurement import Measure
from common.celeryapp import get_celery_app
from common.store.entities import ENTITY_TYPE_MODEL_MAP
from oozer.common.job_scope import JobScope

logger = logging.getLogger(__name__)
app = get_celery_app()

def _extract_tags_for_report_job_status(job_scope: JobScope, *_, **__):
    return {'entity_type': job_scope.entity_type, 'ad_account_id': job_scope.ad_account_id}

@app.task
@Measure.timer(__name__, function_name_as_metric=True, extract_tags_from_arguments=_extract_tags_for_report_job_status)
@Measure.counter(
    __name__, function_name_as_metric=True, extract_tags_from_arguments=_extract_tags_for_report_job_status
)
def set_inaccessible_entity_task(job_scope: JobScope):
    """Update entity record to flag it as inaccessible."""
    report_variant = job_scope.report_variant
    entity_id = job_scope.entity_id

    model_class = ENTITY_TYPE_MODEL_MAP[report_variant]
    model = model_class(page_id=job_scope.ad_account_id, entity_id=entity_id)
    try:
        model.update(actions=[model_class.is_accessible.set(False)])
    except UpdateError as ex:
        if ErrorInspector.is_dynamo_throughput_error(ex):
            logger.info(str(ex))
        else:
            raise


#!/usr/bin/env python3
from sys import stdout
import serial
import time
from threading import Thread
import sys
import blynklib
from enum import Enum
from eventhandler import EventHandler

def char_range(c1, c2):
    """Generates the characters from `c1` to `c2`, inclusive."""
    for c in range(ord(c1), ord(c2)+1):
        yield chr(c)

class VirtualPinMode(Enum):
     NULL = 0
     READ_TIME = 1
     READ_WRITE = 2

#class Protocol(object):
#
#    def _pack_msg(self, msg_type, *args, **kwargs):
#        data = ('\0'.join([str(curr_arg) for curr_arg in args])).encode('utf-8')
#        return struct.pack('!BHH', msg_type, self._get_msg_id(**kwargs), len(data)) + data

class VirtualPin:
    name="s"
    pin=0
    value=None
    mode = VirtualPinMode.NULL

    def __init__(self,pin):
        self.pin=pin

    def get(self):
        return self.value

    def getBool(self):
        return str(self.value)== '1'

    #
    def read(self,con):
        c = "_vr "+ str(self.pin) +"\n"
        con.write(c.encode())

    def write(self,con,val):
        self.value=val
        if (self.mode == VirtualPinMode.NULL):
            self.mode=VirtualPinMode.READ_WRITE

        print ("[CLOUD] " + str(self.pin) +"="+str(self.value))
        self.send(con)

    def out_write(self,con,val):
        self.value=val
        if (self.mode == VirtualPinMode.NULL):
            self.mode=VirtualPinMode.READ_WRITE

        print ("[<< CLOUD] " + str(self.pin) +"="+str(self.value))
        self.send(con)

    def send(self,com):
        #print("_wm")
        c = "_vw "+ str(self.pin) +" "+str(self.value)+"\n"
        aa = c.encode(encoding='ASCII',errors='strict')
        #print(aa)
        com.write(aa)
        #com.write(bytes(c, 'utf-8'))

    def onConnect(self,con):
        if (self.mode == VirtualPinMode.READ_WRITE ):
            self.send(con)

#############
### NOTE: 0 is system message
class SharedMemory:

    pins = {}

    def __init__(self):
        for c in range(0, 128):
            self.pins[c]=VirtualPin(c)

    def get(self,pin):
        return self.pins[pin].get()

    def getBool(self,pin):
        return self.pins[pin].getBool()

    def read(self,con,pin_list):
        for c in pin_list:
           self.pins[c].read(con)

    def write(self,con,pin,val):
        if (pin==0):
            #system message
            self.onConnect(con)
            print("Started ")

            #self.pins[0].com.write("_wm "+self.value+ "\n")
        else:
            self.pins[pin].write(con,val)

    def out_write(self,con,pin,val):
        self.pins[pin].out_write(con,val)

    def onConnect(self,con):
        for key in self.pins:
            #if (self.pins[key].mode == VirtualPinMode.READ_WRITE ):
                self.pins[key].onConnect(con)

class ArduinoClient:

    port = "COM7"
    baud_rate = 9600
    end_param='\x00'
    isOpen=False

    def __init__(self,port,baud_rate,memory):
        self.port=port
        self.baud_rate=baud_rate
        self.memory=memory
        self.event_handler = EventHandler('onConnect', 'onDisconnect')
        #self.event_handler.link(self.__on_newuser_join, 'onConnect')

    def start(self):
        print("Started")
        #self.memory = SharedMemory()

        #t = Thread(target=self.run, args=())
        #t.start()
        #self.run();

    def con(self):
        return self.ser

    def tick(self):

            #while True:
            try:
                if (not self.isOpen):
                    self.ser = serial.Serial(self.port, self.baud_rate, timeout=1)
                    #ser.setDTR(True)
                    self.ser.flush()
                    self.isOpen=True
                    self.event_handler.fire('onConnect',self)
                    print("CLOUD Connected  at ",self.port)
                else:
                    #while True:
                    if self.ser.in_waiting > 0:
                        line = self.ser.readline().decode('utf-8').rstrip()
                        #print(line)
                        if (line.startswith("vw")):
                            #print(repr(line[2]))
                            i = line.index(self.end_param,3)
                            pin = int(line[3:i])
                            #print ("pin = " + str(pin))

                            self.memory.out_write(self.ser,pin,line[i+1:].strip()[:-1])
                        else:
                            print(line)

            except Exception as e:
                if (self.isOpen):
                    self.event_handler.fire('onDisconnect',self)
                self.isOpen=False
                #e = sys.exc_info()[0]
                print("CLOUD Connecting ..",e)
                time.sleep(1)

############################
class BaseBoard:
    timers = []

    def addTimer(self,time_secs,callback):
        self.timers.append({"time_len" : time_secs,"time":time.time(),"callback": callback})

    def __init__(self,client,memory):
        self.memory=memory
        self.client=client

    def write(self,pin,val):
        self.memory.write(self.client.con(),pin,val)

    def tick(self):

        for timer in self.timers:
            delta =   time.time() - timer["time"]
            if (delta > timer["time_len"] ):
                timer["time"]  = time.time()
                #print(delta)
                timer["callback"] ()


import pickle
import numpy as np
import matplotlib.pyplot as plt
import argparse
import re
import os

def parse_arguments():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--experiments',
        help = 'A comma separated list of experiment ids in filesystem',
        type =  lambda x: x.split(','),
    )
    parser.add_argument(
        '--ids',
        help = 'A comma separated list of experiment ids to use as legends',
        type =  lambda x: x.split(','),
    )
    parser.add_argument(
        '--item',
        help = 'Choice of item to plot and compare',
        choices = ['total', 'action', 'mu', 'omega'],
    )
    parser.add_argument(
        '--model',
        help = 'Choice of model to plot losses for',
        choices = ['actor', 'enc'],
    )
    args = parser.parse_args()
    return args

def find_max_epoch(experiment, model):
    path = os.path.join(
        'exp{exp}'.format(exp = experiment),
        'pretrain_{model}'.format(model = model),
    )
    files = os.listdir(path)
    numbers = []
    for f in files:
        if 'action' in f or 'mu' in f or 'omega' in f or 'ckpt' in f or 'png' in f or 'checkpoint' in f:
            continue
        else:
            try:
                numbers.append(int(f[:-7].split('_')[-1]))
            except ValueError:
                print(f)
                pass
    if len(numbers) == 0:
        for f in files:
            if 'ckpt' in f or 'png' in f or 'checkpoint' in f:
                continue
            else:
                try:
                    numbers.append(int(f[:-7].split('_')[-1]))
                except ValueError:
                    print(f)
                    pass
    return max(numbers)

def get_filename(experiment, model, item, epoch):
    if item == 'total':
        item = ''
    else:
        item += '_'
    path = os.path.join(
        'exp{exp}'.format(exp = experiment),
        'pretrain_{model}'.format(model = model),
        'loss_{item}{exp}_pretrain_{model}_{epoch}.pickle'.format(
            item = item,
            exp = experiment,
            model = model,
            epoch = epoch
        )
    )
    return path

def main():
    args = parse_arguments()
    files = []
    epochs = []
    losses = []
    fig, ax = plt.subplots(1,1, figsize = (7,7))
    for i, exp in enumerate(args.experiments):
        epoch = find_max_epoch(exp, args.model)
        epochs.append(epoch)
        f = get_filename(exp, args.model, args.item, epoch)
        pkl = open(f, 'rb')
        loss = pickle.load(pkl)
        pkl.close()
        model = args.model
        if model == 'enc':
            model = 'encoder'
        ax.plot(loss, label = 'Experiment {exp} {model} {item} mse loss'.format(
            exp = args.ids[i],
            model = model,
            item = args.item
        ))
    ax.legend(loc = 'upper right')
    path = 'loss_pretrain_{model}_{item}_experiments'.format(
        model = args.model,
        item = args.item
    )
    for exp in args.experiments:
        path += '_{exp}'.format(exp = exp)
    path += '.png'
    fig.savefig(path)
    plt.show()
    plt.close()

if __name__ == '__main__':
    main()


from environments.MyFrozenLakeEnv import MyFrozenLake
from environments.MyFrozenLake8Env import MyFrozenLake8

import numpy as np
from tqdm import tqdm
import gym
import torch
import torch.nn as nn
import torch.nn.functional as F
import matplotlib.pyplot as plt
import os
import imageio
from datetime import datetime

L = 64
M = 4
alpha = 0.456
# N = 30
traj_len = 300
dim = int(np.sqrt(L))

log = str(datetime.now().strftime("%Y-%m-%d_%H-%M-%S"))
base_dir = "results8"

os.makedirs(base_dir, exist_ok=True)
absorbing_goal = False
save = True
if save:
    os.makedirs(base_dir+"/"+log, exist_ok=True)

class STL():
    def __init__(self):
        self.h1_mid_state = L//2
        self.h2_mid_state = L//2

    def get_dist(self, state1, state2):
        x1 = state2 % dim
        y1 = state2 // dim

        x2 = state1 % dim
        y2 = state1 // dim

        return abs(x1-x2) + abs(y1-y2)

    def get_min_hole_dist(self, state):

        # mid for hole 1
        min_dist3 = 9999
        dist_to_hole = self.get_dist(self.h1_mid_state, state)
        min_dist3 = min(min_dist3, dist_to_hole)

        # mid for hole 2
        dist_to_hole = self.get_dist(self.h2_mid_state, state)
        min_dist3 = min(min_dist3, dist_to_hole)

        return min_dist3

    def calc_costs(self,traj):
        min_dist_to_hole_mid = 9999

        for state in traj:
            step_cost_mid = self.get_min_hole_dist(state)
            min_dist_to_hole_mid = min(min_dist_to_hole_mid, step_cost_mid)
        return min_dist_to_hole_mid

losses = []
Ns = []
for N in range(14, 184):
    if N % 5 == 0:
        print(N)

        # State-Action visitation frequency
        K = np.zeros((N, L*M))

        # Next - State visitation frequency
        H = np.zeros((N, L))

        env = MyFrozenLake8(probabilistic=False, absorbing_goal_state = absorbing_goal)
        C = []
        R = []
        Z = []
        trajs = []

        for j in range(N):
            # Step1 - Create a random trajectory
            # action_seq = list(np.random.choice([0,1,2,3], 30, p = [0.1,0.4,0.4,0.1]))
            action_seq = list(np.random.choice([0,1,2,3], traj_len))
            state_seq = []

            k = 0
            score = 0
            min_dist_to_hole = 9999

            st = env.reset()
            while k < len(action_seq):
                action = action_seq[k]
                k += 1
                ij_index = st * M + int(action)
                K[j,ij_index] += 1

                next_st, reward, done, info = env.step(action)
                H[j,next_st] += 1

                min_dist_to_hole = min(min_dist_to_hole, info["min_hole_dist"])

                # if next_st in env.constraint_states:
                score += reward
                state_seq.append(st)
                st = next_st

            n_score = np.random.normal(score, 0.5)
            R.append(n_score)
            # min_dist_to_hole = np.random.normal(min_dist_to_hole, 0.5)
            C.append(min_dist_to_hole)
            trajs.append(state_seq)
            z = score-alpha*min_dist_to_hole
            # n_z = np.random.normal(z, 0.5)
            Z.append(z)

            # if j%100 == 0:
            #     print("\n")
            #     print(j)

        R = np.array(R)
        C = np.array(C)
        Z = np.array(Z)

        #####################################################################################################################
        # Section to calc stl specifications
        #####################################################################################################################

        stl = STL()
        h1_upper = L
        h1_lower = 0
        h2_upper = L
        h2_lower = 0
        for j in range(10):

            # Get loss if h1 is upper limit, h2 is upper limit,
            stl.h1_mid_state = h1_upper
            stl.h2_mid_state = h2_upper
            loss1 = 0
            for i, traj in enumerate(trajs):
                cost_actual = C[i]
                cost_mid_est = stl.calc_costs(traj)
                loss1 += (cost_mid_est-cost_actual)**2

            # Get loss if h1 is upper limit, h2 is lower limit,
            stl.h1_mid_state = h1_upper
            stl.h2_mid_state = h2_lower
            loss2 = 0
            for i, traj in enumerate(trajs):
                cost_actual = C[i]
                cost_mid_est = stl.calc_costs(traj)
                loss2 += (cost_mid_est-cost_actual)**2

            # Get loss if h1 is lower limit, h2 is upper limit,
            stl.h1_mid_state = h1_lower
            stl.h2_mid_state = h2_upper
            loss3 = 0
            for i, traj in enumerate(trajs):
                cost_actual = C[i]
                cost_mid_est = stl.calc_costs(traj)
                loss3 += (cost_mid_est-cost_actual)**2

            # Get loss if h1 is lower limit, h2 is lower limit,
            stl.h1_mid_state = h1_lower
            stl.h2_mid_state = h2_lower
            loss4 = 0
            for i, traj in enumerate(trajs):
                cost_actual = C[i]
                cost_mid_est = stl.calc_costs(traj)
                loss4 += (cost_mid_est-cost_actual)**2

            if min([loss1, loss2, loss3, loss4]) == loss1:
                h1_lower = (h1_lower + h1_upper)//2
                h2_lower = (h2_lower + h2_upper)//2

            elif min([loss1, loss2, loss3, loss4]) == loss2:
                h1_lower = (h1_lower + h1_upper)//2
                h2_upper = (h2_lower + h2_upper)//2

            elif min([loss1, loss2, loss3, loss4]) == loss3:
                h1_upper = (h1_lower + h1_upper)//2
                h2_lower = (h2_lower + h2_upper)//2

            elif min([loss1, loss2, loss3, loss4]) == loss4:
                h1_upper = (h1_lower + h1_upper)//2
                h2_upper = (h2_lower + h2_upper)//2
            # print(min([loss1, loss2, loss3, loss4]))

        print("Holes are in:{} and {}".format((h1_upper + h1_lower)//2, (h2_upper + h2_lower)//2))

        #####################################################################################################################

        sym_mat1 = np.dot(K.transpose(), K)
        pseudo_inv1 = np.dot(np.linalg.inv(sym_mat1), K.transpose())
        r1 = np.dot(pseudo_inv1, R)

        sym_mat2 = np.dot(H.transpose(), H)
        pseudo_inv2 = np.dot(np.linalg.inv(sym_mat2), H.transpose())
        r2 = np.dot(pseudo_inv2, R)

        # H_dash = np.concatenate((H,C.reshape(N,1)), axis = 1)
        # sym_mat3 = np.dot(H_dash.transpose(), H_dash)
        # pseudo_inv3 = np.dot(np.linalg.inv(sym_mat3), H_dash.transpose())
        # r3 = np.dot(pseudo_inv3, Z)

        # print(r3[-1])

        if save:
            mat1 = np.zeros((dim,dim))
            mat2 = np.zeros((dim,dim))
            for state in env.state_rewards:

                true_reward = env.state_rewards[state]
                predicted_reward = r2[state]

                mat1[state//dim, state%dim] = true_reward
                mat2[state//dim, state%dim] = predicted_reward

            fig2, (ax3, ax4, ax5) = plt.subplots(nrows=1, ncols=3, figsize=(15, 10))

            ax3.title.set_text('reward true')
            plot1 = ax3.imshow(mat1, label='reward true', vmin = 0, vmax = 0.6)

            ax4.title.set_text('reward predicted')
            plot2 = ax4.imshow(mat2, label='reward predicted', vmin = 0, vmax = 0.6)

            ax5.title.set_text('error')
            plot3 = ax5.imshow(mat2-mat1, label='error', vmin = -0.5, vmax = 0.5)

            sum_squared_errors = np.sum((mat2-mat1)**2)
            losses.append(sum_squared_errors)
            Ns.append(N)

            fig2.colorbar(plot1,ax=ax3)
            fig2.colorbar(plot2,ax=ax4)
            fig2.colorbar(plot3,ax=ax5)

            fig2.suptitle("N = " + str(N) + "; error = " + str(sum_squared_errors))

            if absorbing_goal:
                fig2.savefig(base_dir + '/'+ log +'/reward_matrix_'+str(N)+'_absorbing.png')
            else:
                fig2.savefig(base_dir + '/'+ log +'/reward_matrix_'+str(N)+'_not_absorbing.png')

if save:
    sorted_file_names = {}
    filenames = os.listdir(base_dir + "/"+log)

    for filename in filenames:
        N = int(filename.split("_")[2])
        sorted_file_names[N] = filename

    sorted_file_names = sorted_file_names.items()

    sorted_file_names = sorted(sorted_file_names)

    images = []
    for N, filename in sorted_file_names:
        images.append(imageio.imread(base_dir + "/"+log+"/"+filename))
    imageio.mimsave(base_dir + '/'+ log +'/rewards.gif', images)

    fig3, ax6 = plt.subplots( nrows=1, ncols=1 )  # create figure & 1 axis
    ax6.plot(np.array(Ns),np.array(losses))
    ax6.set(xlabel='Number of trajectories', ylabel='Error')

    fig3.savefig(base_dir + '/'+ log +'/loss_curve.png')

print("Done")


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import logging
import logging.config
import os
import sys
import socket
import errno

import common
from autotest_lib.client.common_lib import autotest_enum
from autotest_lib.client.common_lib import logging_manager
from autotest_lib.server import server_logging_config
from autotest_lib.server.hosts import factory
from autotest_lib.server.hosts import servo_host

import verifiers

RETURN_CODES = autotest_enum.AutotestEnum(
        'OK',
        'VERIFY_FAILURE',
        'OTHER_FAILURES'
)

ACTION_VERIFY_DUT_STORAGE = 'verify-dut-storage'
ACTION_VERIFY_SERVO_USB = 'verify-servo-usb-drive'
ACTION_VERIFY_SERVO_FW = 'verify-servo-fw'
ACTION_FLASH_SERVO_KEYBOARD_MAP = 'flash-servo-keyboard-map'
ACTION_VERIFY_DUT_MACADDR = 'verify-dut-macaddr'
ACTION_VERIFY_RPM_CONFIG = 'verify-rpm-config'

_LOG_FILE = 'audit.log'
_SERVO_UART_LOGS = 'servo_uart'

VERIFIER_MAP = {
        ACTION_VERIFY_DUT_STORAGE: verifiers.VerifyDutStorage,
        ACTION_VERIFY_SERVO_USB: verifiers.VerifyServoUsb,
        ACTION_VERIFY_SERVO_FW: verifiers.VerifyServoFw,
        ACTION_FLASH_SERVO_KEYBOARD_MAP:
        verifiers.FlashServoKeyboardMapVerifier,
        ACTION_VERIFY_DUT_MACADDR: verifiers.VerifyDUTMacAddress,
        ACTION_VERIFY_RPM_CONFIG: verifiers.VerifyRPMConfig,
}

# Actions required Servod service
ACTIONS_REQUIRED_SERVOD = set([
        ACTION_VERIFY_DUT_STORAGE,
        ACTION_VERIFY_SERVO_USB,
        ACTION_FLASH_SERVO_KEYBOARD_MAP,
        ACTION_VERIFY_DUT_MACADDR,
])

# Actions required ServoHost without Servod process
ACTIONS_REQUIRED_SERVO_HOST = set([
    ACTION_VERIFY_SERVO_FW,
])

class DutAuditError(Exception):
    """Generic error raised during DUT audit."""

def main():
    """Tool to audit a DUT."""
    opts = _parse_args()

    # Create logging setting
    logging_manager.configure_logging(
        server_logging_config.ServerLoggingConfig(),
        results_dir=opts.results_dir)

    logging.debug('autoserv is running in drone %s.', socket.gethostname())
    logging.debug('audit environment: %r', os.environ)
    logging.debug('audit command was: %s', ' '.join(sys.argv))
    logging.debug('audit parsed options: %s', opts)

    # Initialize ServoHost without running Servod process.
    need_servo_host = bool(set(opts.actions) & ACTIONS_REQUIRED_SERVO_HOST)
    # Initialize ServoHost with running Servod process.
    need_servod = bool(set(opts.actions) & ACTIONS_REQUIRED_SERVOD)

    # Create folder for servo uart logs.
    servo_uart_logs_dir = None
    if need_servod:
        servo_uart_logs_dir = _create_servo_uart_path(opts.results_dir)

    try:
        host_object = factory.create_target_host(
                opts.hostname,
                host_info_path=opts.host_info_file,
                try_lab_servo=need_servod,
                try_servo_repair=need_servod,
                servo_uart_logs_dir=servo_uart_logs_dir)
    except Exception as err:
        logging.error("fail to create host: %s", err)
        return RETURN_CODES.OTHER_FAILURES

    with host_object as host:
        if need_servo_host and not need_servod:
            try:
                host.set_servo_host(servo_host.ServoHost(
                    **servo_host.get_servo_args_for_host(host)
                ))
            except Exception as err:
                logging.error("fail to init servo host: %s", err)
                return RETURN_CODES.OTHER_FAILURES
        for action in opts.actions:
            if opts.dry_run:
                logging.info('DRY RUN: Would have run actions %s', action)
                return

            response = _verify(action, host, opts.results_dir)
            if response:
                return response

    return RETURN_CODES.OK

def _verify(action, host, resultdir):
    """Run verifier for the action with targeted host.

    @param action: The action requested to run the verifier.
    @param host: The host presentation of the DUT.
    """
    try:
        _log("START", action)
        verifier = VERIFIER_MAP[action]
        if verifier:
            v = verifier(host)
            v.set_result_dir(resultdir)
            v.verify()
        else:
            logging.info('Verifier is not specified')
        _log("END_GOOD", action)
    except Exception as err:
        _log("END_FAIL", action, err)
        return RETURN_CODES.VERIFY_FAILURE

def _log(status, action, err=None):
    if err:
        message = '%s:%s; %s' % (action, status, str(err))
    else:
        message = '%s:%s' % (action, status)
    logging.info(message)

def _create_servo_uart_path(results_dir):
    servo_uart_logs = os.path.join(results_dir, _SERVO_UART_LOGS)
    try:
        if not os.path.exists(servo_uart_logs):
            os.makedirs(servo_uart_logs)
    except OSError as e:
        logging.debug(
                '(not critical) Fail to create dir for servo logs;'
                ' %s', e)
        if not (e.errno == errno.EEXIST):
            servo_uart_logs = None
    return servo_uart_logs

def _parse_args():
    parser = argparse.ArgumentParser(description='Audit DUT in a lab.')

    parser.add_argument(
            'actions',
            nargs='+',
            choices=list(VERIFIER_MAP),
            help='DUT audit actions to execute.',
    )
    parser.add_argument(
            '--dry-run',
            action='store_true',
            default=False,
            help='Run in dry-run mode. No changes will be made to the DUT.',
    )
    parser.add_argument(
            '--results-dir',
            required=True,
            help='Directory to drop logs and output artifacts in.',
    )

    parser.add_argument(
            '--hostname',
            required=True,
            help='Hostname of the DUT to audit.',
    )
    parser.add_argument(
            '--host-info-file',
            required=True,
            help=('Full path to HostInfo file.'
                  ' DUT inventory information is read from the HostInfo file.'
                  ),
    )

    return parser.parse_args()

if __name__ == '__main__':
    sys.exit(main())


import functools as fp
from typing import List

from chess.pieces.base import ChessPiece
from chess.constants import (
    FigureType, Direction, FigureColor, Rank
)
from chess.position import Position
from chess.utils import prune_moves_if_king_in_check

from functional import seq

class Pawn(ChessPiece):
    figure_type = FigureType.PAWN

    @property
    def is_white(self):
        return self.color == FigureColor.WHITE

    def is_at_starting_pos(self, position: Position):
        starting_rank = Rank.TWO if self.is_white else Rank.SEVEN
        return position.rank == starting_rank

    def get_possible_positions(self, position):
        rank, file = position.rank, position.file
        forward = Direction.UP if self.is_white else Direction.DOWN

        return {
            'one_forward': Position(rank + forward, file),
            'two_forward': Position(rank + 2 * forward, file),
            'attack_right': Position(rank + forward, file + Direction.RIGHT),
            'attack_left': Position(rank + forward, file + Direction.LEFT),
        }

    # NOTE: ideally this should be static or class method but it isnt
    # because of is_white check
    @prune_moves_if_king_in_check
    def generate_moves(self, board, pawn_pos: Position) -> List[Position]:
        potential_positions = self.get_possible_positions(pawn_pos)

        left_attack  = potential_positions['attack_left']
        right_attack = potential_positions['attack_right']
        one_forward  = potential_positions['one_forward']
        two_forward  = potential_positions['two_forward']

        positions = []

        if board.is_in_bounds(one_forward) and board.is_empty(one_forward):
            positions.append(one_forward)

        if self.is_at_starting_pos(pawn_pos) and board.is_empty(two_forward):
            positions.append(two_forward)

        can_attack = fp.partial(self._can_attack, board, pawn_pos)

        if can_attack(left_attack):
            positions.append(left_attack)

        if can_attack(right_attack):
            positions.append(right_attack)

        return positions

    @staticmethod
    def _can_attack(board, from_pos, attack_pos):
        return board.is_in_bounds(attack_pos) and board.are_enemies(from_pos, attack_pos) or board.en_passant_pos == attack_pos


from django.shortcuts import resolve_url
from django.test.client import RequestFactory

import pytest

from api.admin.site import openverse_admin

class MockUser:
    is_active = True
    is_staff = True
    pk = 1

    def has_perm(self, *args):
        return True

    def has_module_perms(self, *args):
        return True

@pytest.mark.parametrize(
    "environment_value, should_be_present",
    [
        ("staging", True),
        ("local", True),
        ("production", False),
    ],
)
def test_admin_base_html_renders_message(environment_value, should_be_present, db):
    url = resolve_url("admin:index")
    request = RequestFactory().get(url)
    request.user = MockUser()

    index = openverse_admin.index(request)
    index.context_data["ENVIRONMENT"] = environment_value
    html = index.rendered_content

    assert (
        "Next staging database restore will occur in" in html
    ) == should_be_present, (
        "Rendered message about staging database restore was missing or "
        "incorrectly present in Admin UI"
    )