from __future__ import annotations

from dataclasses import dataclass
from dataclasses import field

import symforce.symbolic as sf
from symforce import jacobian_helpers
from symforce import typing as T

@dataclass
class ResidualBlock:
    """
    A single residual vector, with associated extra values.  The extra values are not used in the
    optimization, but are intended to be additional outputs used for debugging or other purposes.
    """

    residual: sf.Matrix
    extra_values: T.Optional[T.Dataclass] = None
    metadata: T.Optional[T.Dict[str, T.Any]] = None

    def compute_jacobians(
        self,
        inputs: T.Sequence[T.Element],
        residual_name: str = None,
        key_names: T.Sequence[str] = None,
    ) -> T.Sequence[sf.Matrix]:
        """
        Compute the jacobians of this residual block with respect to a sequence of inputs

        Args:
            inputs: Sequence of inputs to differentiate with respect to
            residual_name: Optional human-readable name of the residual to be used for debug
                           messages
            key_names: Optional sequence of human-readable names for the inputs to be used for debug
                       messages

        Returns:
            Sequence of jacobians of the residual with respect to each entry in inputs
        """
        return jacobian_helpers.tangent_jacobians(self.residual, inputs)

    def set_metadata(self, key: str, value: T.Any) -> ResidualBlock:
        """
        Sets a metadata field of the residual block and returns the residual block.
        """
        if self.metadata is None:
            self.metadata = {key: value}
        else:
            self.metadata[key] = value
        return self

@dataclass
class ResidualBlockWithCustomJacobian(ResidualBlock):
    """
    A residual block, with a custom jacobian for the residual

    This should generally only be used if you want to override the jacobian computed by SymForce,
    e.g. to stop derivatives with respect to certain variables or directions, or because the
    jacobian can be analytically simplified in a way that SymForce won't do automatically.  The
    ``custom_jacobians`` field should then be filled out with a mapping from all inputs to the
    residual which may be differentiated with respect to, to the desired jacobian of the residual
    with respect to each of those inputs.
    """

    custom_jacobians: T.Dict[T.Element, sf.Matrix] = field(default_factory=dict)

    def compute_jacobians(
        self,
        inputs: T.Sequence[T.Element],
        residual_name: str = None,
        key_names: T.Sequence[str] = None,
    ) -> T.Sequence[sf.Matrix]:
        """
        Compute the jacobians of this residual block with respect to a sequence of inputs

        Args:
            inputs: Sequence of inputs to differentiate with respect to
            residual_name: Optional human-readable name of the residual to be used for debug
                           messages
            key_names: Optional sequence of human-readable names for the inputs to be used for debug
                       messages

        Returns:
            Sequence of jacobians of the residual with respect to each entry in inputs
        """
        residual_jacobians = []
        for i, input_element in enumerate(inputs):
            if input_element in self.custom_jacobians:
                # The user provided a derivative with respect to this input
                residual_jacobians.append(self.custom_jacobians[input_element])
            else:
                # The user did not provide a derivative with respect to this input.  So,
                # compute it.  If it's nonzero, raise an error, since the user probably
                # wants to provide custom jacobians for all the variables if they
                # provided one
                residual_input_jacobian = self.residual.jacobian(input_element)
                if (
                    residual_input_jacobian
                    != sf.matrix_type_from_shape(residual_input_jacobian.shape).zero()
                ):
                    residual_name = residual_name or str(self)

                    if key_names is not None:
                        key_name = key_names[i]
                    else:
                        key_name = str(input_element)

                    raise ValueError(
                        f"The residual `{residual_name}` has a nonzero jacobian with respect to "
                        f"input `{key_name}`.  Custom jacobians were provided for this residual, "
                        "but not for this input variable.  If you wish to use the automatically "
                        "computed jacobian for this input, please compute it using "
                        "`jacobian_helpers.tangent_jacobians(residual, [input])[0]` and add it to "
                        "the custom_jacobians dictionary"
                    )
                residual_jacobians.append(residual_input_jacobian)
        return residual_jacobians


"""
An RPC server base class for encapsulating a service class and receiving requests that
will call that encapsulated class. This is part of a remote service that communicates
with the projectServer.
"""
import os
import sys
import re
import time
import json
import pickle
import logging
import argparse
import threading
import websocket
from base64 import b64encode, b64decode
from rtCommon.remoteable import RemoteHandler
from rtCommon.projectUtils import generateDataParts
from rtCommon.utils import DebugLevels, trimDictBytes
from rtCommon.errors import RequestError
from rtCommon.projectUtils import login, certFile, checkSSLCertAltName, makeSSLCertFile

class WsRemoteService:
    remoteHandler = RemoteHandler()
    commLock = threading.Lock()
    shouldExit = False

    def __init__(self, args, channelName):
        """
        Args:
            args: Argparse args for establishing a connection to the projectServer
            channelName: the websocket channel to connect on, e.g. 'wsData'
        """
        self.channelName = channelName
        self.args = args
        self.sessionCookie = None
        self.needLogin = True
        self.started = False

        # # Starts the receiver in it's own thread
        # self.recvThread = threading.Thread(name='recvThread', target=self.wsReceiver)
        # self.recvThread.setDaemon(True)
        # self.recvThread.start()

    def addHandlerClass(self, classType, classInstance):
        """Register the class that will handle the received requests via the class type"""
        WsRemoteService.remoteHandler.registerClassInstance(classType, classInstance)

    def addHandlerClassName(self, className, classInstance):
        """Register the class that will handle the received requests via the class name"""
        WsRemoteService.remoteHandler.registerClassNameInstance(className, classInstance)

    def runForever(self):
        """Run the receiver loop. This function doesn't return."""
        # go into loop trying to do webSocket connection periodically
        args = self.args
        # print(f'args {self.args}')
        WsRemoteService.shouldExit = False
        while not WsRemoteService.shouldExit:
            try:
                if self.needLogin or self.sessionCookie is None:
                    self.sessionCookie = login(args.server, args.username, args.password, testMode=args.test)
                wsAddr = os.path.join('wss://', args.server, self.channelName)
                if args.test:
                    print("Warning: using non-encrypted connection for test mode")
                    wsAddr = os.path.join('ws://', args.server, self.channelName)
                logging.log(DebugLevels.L6, "Trying connection: %s", wsAddr)
                ws = websocket.WebSocketApp(wsAddr,
                                            on_message=WsRemoteService.on_message,
                                            on_close=WsRemoteService.on_close,
                                            on_error=WsRemoteService.on_error,
                                            cookie="login="+self.sessionCookie)
                logging.log(logging.INFO, "Connected to: %s", wsAddr)
                print("Connected to: {}".format(wsAddr))
                self.started = True
                ws.run_forever(sslopt={"ca_certs": certFile})
            except Exception as err:
                logging.log(logging.INFO, "WsRemoteService Exception {}: {}".format(type(err).__name__, str(err)))
            print('sleep {}'.format(args.interval))
            time.sleep(args.interval)

    @staticmethod
    def stop():
        WsRemoteService.shouldExit = True

    @staticmethod
    def send_response(client, response):
        WsRemoteService.commLock.acquire()
        try:
            client.send(json.dumps(response))
        finally:
            WsRemoteService.commLock.release()

    @staticmethod
    def on_message(client, message):
        """
        Main message dispatcher that will get the request from projectServer
        and call the registered handler to process the request. It will then
        return the result of the handler function back to the projectServer.
        """
        response = {'status': 400, 'error': 'unhandled request'}
        cmd = 'unknown'
        try:
            request = json.loads(message)
            request = decodeByteTypeArgs(request)
            # print(f'on_message: message {request} type: {type(request)}')
            # create the response message but without data objects
            response = request.copy()
            response.pop('data', None)
            response.pop('args', None)
            response.pop('kwargs', None)
            trimDictBytes(response)
            cmd = request.get('cmd')
            # decode any encoded byte args
            # TODO - spin off a thread for each request passing in client so the thread call client.send
            callResult = WsRemoteService.remoteHandler.runRemoteCall(request)
            # serialize the callResult
            # print(f'callresult type: {type(callResult)}')
            if isNativeType(callResult):
                if type(callResult) == bytes:
                    data = callResult
                    response['dataSerialization'] = 'bytes'
                else:
                    # encode to json and then as a byte string
                    data = json.dumps(callResult).encode()
                    response['dataSerialization'] = 'json'
            else:
                data = pickle.dumps(callResult)
                response['dataSerialization'] = 'pickle'
            # return callResult
            response['status'] = 200
            compress = False
            if len(data) > 1024 * 1024:
                compress = True
            for msgPart in generateDataParts(data, response, compress=compress):
                WsRemoteService.send_response(client, msgPart)
        except Exception as err:
            errStr = "RPC Exception: {}: {}".format(cmd, err)
            print(errStr)
            response.update({'status': 400, 'error': errStr})
            WsRemoteService.send_response(client, response)
            if cmd == 'error':
                sys.exit()
            return

    @staticmethod
    def on_error(client, error):
        if type(error) is KeyboardInterrupt:
            WsRemoteService.shouldExit = True
        else:
            logging.log(logging.WARNING, "on_error: WsRemoteService: {} {}".
                        format(type(error), str(error)))

    @staticmethod
    def on_close(client, code, reason):
        print('## Connection closed, check if projectServer allows remote services.')
        print('## May need to restart projectServer with --dataRemote --subjectRemote options.')
        logging.info(f'Connection closed {code} {reason}')

def isNativeType(var):
    nativeTypes = (int, float, str, bytes, list, dict, set, tuple, bytearray, memoryview, range, complex)
    if var in nativeTypes:
        return True
    return False

def encodeByteTypeArgs(cmd) -> dict:
    """
    Check if any args are of type 'bytes' and if so base64 encode them.
    The original arg will be replaced with a tag that will reference the encoded bytes
        within the cmd dict.
    Args:
        cmd: a dictionary of the command to check
    Returns:
        A cmd dictionary with the byte args encoded
    """
    args = cmd.get('args', ())
    byteArgIndices = []
    for i, arg in enumerate(args):
        if type(arg) is bytes:
            byteArgIndices.append(i)

    if len(byteArgIndices) != 0:
        # convert args from tuple to list so we can modify it
        args = list(args)
        for i in byteArgIndices:
            # encode as base64 and put a tag holder in place
            encdata = b64encode(args[i]).decode('utf-8')
            tag = 'encodedBytes_' + str(i)
            args[i] = tag
            cmd[tag] = encdata
        cmd['args'] = tuple(args)
        cmd['encodedByteArgs'] = byteArgIndices

    # Check and encode keyword args also
    kwargs = cmd.get('kwargs', {})
    byteKwargKeys = []
    for key, arg in kwargs.items():
        if type(arg) is bytes:
            byteKwargKeys.append(key)
            encdata = b64encode(arg).decode('utf-8')
            kwargs[key] = encdata
    if len(byteKwargKeys) != 0:
        cmd['encodedByteKwargs'] = byteKwargKeys
        cmd['kwargs'] = kwargs

    return cmd

def decodeByteTypeArgs(cmd) -> dict:
    """
    Decodes rpc args that were previously encoded with encodeByteTypeArgs.
    Args:
        cmd: a dictionary with encoded args
    Returns:
        cmd: a dictionary with decoded args
    """
    byteArgIndices = cmd.get('encodedByteArgs')
    if byteArgIndices is not None:
        args = cmd.get('args', ())
        args = list(args)
        for i in byteArgIndices:
            tag = 'encodedBytes_' + str(i)
            encdata = cmd.get(tag)
            if encdata is None or args[i] != tag:
                raise RequestError(f'Byte encoded data error: index {i} tag {tag}')
            decodedData = b64decode(encdata)
            args[i] = decodedData
            cmd.pop(tag, None)
        cmd.pop('encodedByteArgs')
        cmd['args'] = tuple(args)
    return cmd

def parseConnectionArgs():
    # parse args for client to connect to server
    # argv = sys.argv[1:]
    parser = argparse.ArgumentParser()
    parser.add_argument('-s', action="store", dest="server", default="localhost:8888",
                        help="Server Address with Port [server:port]")
    parser.add_argument('-i', action="store", dest="interval", type=int, default=5,
                        help="Retry connection interval (seconds)")
    parser.add_argument('-u', '--username', action="store", dest="username", default=None,
                        help="rtcloud website username")
    parser.add_argument('-p', '--password', action="store", dest="password", default=None,
                        help="rtcloud website password")
    parser.add_argument('--test', default=False, action='store_true',
                        help='Use unsecure non-encrypted connection')
    args, _ = parser.parse_known_args()

    if not re.match(r'.*:\d+', args.server):
        print("Error: Expecting server address in the form <servername:port>")
        parser.print_help()
        sys.exit()

    # Check if the ssl certificate is valid for this server address
    addr, _ = args.server.split(':')
    if checkSSLCertAltName(certFile, addr) is False:
        # Addr not listed in sslCert, recreate ssl Cert
        makeSSLCertFile(addr)

    return args


import os
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
devicess = [0]
from collections import defaultdict
import time
import argparse
import numpy as np
from PIL import Image
import torch
from sklearn import metrics
from torch.utils.data import DataLoader
from torch.utils.tensorboard import SummaryWriter
from torch import nn
from torchvision import transforms
import torch.distributed as dist
import math
import torchio
from torchio.transforms import (
    ZNormalization,
)
from tqdm import tqdm
from torchvision import utils
from hparam import hparams as hp
from utils.metric import metric
from torch.optim.lr_scheduler import ReduceLROnPlateau,StepLR,CosineAnnealingLR
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

source_train_0_dir = hp.source_train_0_dir
source_train_1_dir = hp.source_train_1_dir

source_test_0_dir = hp.source_test_0_dir
source_test_1_dir = hp.source_test_1_dir

def parse_training_args(parser):
    """
    Parse commandline arguments.
    """

    parser.add_argument('-o', '--output_dir', type=str, default=hp.output_dir, required=False, help='Directory to save checkpoints')
    parser.add_argument('--latest-checkpoint-file', type=str, default=hp.latest_checkpoint_file, help='Store the latest checkpoint in each epoch')

    # training
    training = parser.add_argument_group('training setup')
    training.add_argument('--epochs', type=int, default=hp.total_epochs, help='Number of total epochs to run')
    training.add_argument('--epochs-per-checkpoint', type=int, default=hp.epochs_per_checkpoint, help='Number of epochs per checkpoint')
    training.add_argument('--batch', type=int, default=hp.batch_size, help='batch-size')
    parser.add_argument(
        '-k',
        "--ckpt",
        type=str,
        default=hp.ckpt,
        help="path to the checkpoints to resume training",
    )
    parser.add_argument("--init-lr", type=float, default=hp.init_lr, help="learning rate")
    # TODO
    parser.add_argument(
        "--local_rank", type=int, default=0, help="local rank for distributed training"
    )

    training.add_argument('--amp-run', action='store_true', help='Enable AMP')
    training.add_argument('--cudnn-enabled', default=True, help='Enable cudnn')
    training.add_argument('--cudnn-benchmark', default=True, help='Run cudnn benchmark')
    training.add_argument('--disable-uniform-initialize-bn-weight', action='store_true', help='disable uniform initialization of batchnorm layer weight')

    return parser

def train():

    parser = argparse.ArgumentParser(description='PyTorch Medical Segmentation Training')
    parser = parse_training_args(parser)
    args, _ = parser.parse_known_args()

    args = parser.parse_args()

    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.enabled = args.cudnn_enabled
    torch.backends.cudnn.benchmark = args.cudnn_benchmark

    from data_function import MedData_train
    os.makedirs(args.output_dir, exist_ok=True)

    if hp.mode == '2d':
        # from models.two_d.alexnet import alexnet
        # model = alexnet(num_classes=2)

        # from models.two_d.densenet import densenet121
        # model = densenet121(num_class=2)

        from models.two_d.googlenet import googlenet
        model = googlenet(num_class=2)

        # from models.two_d.mobilenet import mobilenet
        # model = mobilenet(class_num=2)

        # from models.two_d.nasnet import nasnet
        # model = nasnet(class_num=2)

        # from models.two_d.resnet import resnet101
        # model = resnet101(num_classes=2)

        # from models.two_d.resnext import resnext101
        # model = resnext101(class_names=2)

        # from models.two_d.vggnet import vgg16_bn
        # model = vgg16_bn(num_class=2)

    elif hp.mode == '3d':

        from models.three_d.resnet3d import generate_model
        model = generate_model(18,n_input_channels=1,n_classes=2)

        from models.three_d.resnext3d import generate_model
        model = generate_model(50,n_input_channels=1,n_classes=2)

        from models.three_d.densenet3d import generate_model
        model = generate_model(121,n_input_channels=1,num_classes=2)

    model = torch.nn.DataParallel(model, device_ids=devicess)
    optimizer = torch.optim.Adam(model.parameters(), lr=args.init_lr)

    # scheduler = ReduceLROnPlateau(optimizer, 'min',factor=0.5, patience=20, verbose=True)
    scheduler = StepLR(optimizer, step_size=hp.scheduer_step_size, gamma=hp.scheduer_gamma)
    # scheduler = CosineAnnealingLR(optimizer, T_max=50, eta_min=5e-6)

    if args.ckpt is not None:
        print("load model:", args.ckpt)
        print(os.path.join(args.output_dir, args.latest_checkpoint_file))
        ckpt = torch.load(os.path.join(args.output_dir, args.latest_checkpoint_file), map_location=lambda storage, loc: storage)

        model.load_state_dict(ckpt["model"])
        optimizer.load_state_dict(ckpt["optim"])

        for state in optimizer.state.values():
            for k, v in state.items():
                if torch.is_tensor(v):
                    state[k] = v.cuda()

        # scheduler.load_state_dict(ckpt["scheduler"])
        elapsed_epochs = ckpt["epoch"]
    else:
        elapsed_epochs = 0

    model.cuda()

    from loss_function import Classification_Loss
    criterion = Classification_Loss().cuda()

    writer = SummaryWriter(args.output_dir)

    train_dataset = MedData_train(source_train_0_dir,source_train_1_dir)
    train_loader = DataLoader(train_dataset.training_set,
                            batch_size=args.batch,
                            shuffle=True,
                            pin_memory=True,
                            drop_last=True)

    model.train()

    epochs = args.epochs - elapsed_epochs
    iteration = elapsed_epochs * len(train_loader)

    for epoch in range(1, epochs + 1):
        print("epoch:"+str(epoch))
        epoch += elapsed_epochs

        num_iters = 0

        gts = []
        predicts = []
        for i, batch in enumerate(train_loader):

            if hp.debug:
                if i >=1:
                    break

            print(f"Batch: {i}/{len(train_loader)} epoch {epoch}")

            optimizer.zero_grad()

            x = batch['source']['data']
            y = batch['label']

            x = x.type(torch.FloatTensor).cuda()
            y = y.type(torch.LongTensor).cuda()

            if hp.mode == '2d':
                x = x.squeeze(-1)
                x = x[:,:1,:,:]

            outputs = model(x)

            outputs_logit = outputs.argmax(dim=1)

            loss = criterion(outputs, y, model)

            num_iters += 1
            loss.backward()

            optimizer.step()
            iteration += 1

            print("loss:"+str(loss.item()))
            writer.add_scalar('Training/Loss', loss.item(),iteration)

            predicts.append(outputs_logit.cpu().detach().numpy())
            gts.append(y.cpu().detach().numpy())

        predicts = np.concatenate(predicts).flatten().astype(np.int16)
        gts = np.concatenate(gts).flatten().astype(np.int16)

        print(metrics.confusion_matrix(predicts, gts))
        acc = metrics.accuracy_score(predicts, gts)
        recall = metrics.recall_score(predicts, gts)
        f1 = metrics.f1_score(predicts, gts)
        writer.add_scalar('Training/acc', acc,epoch)
        writer.add_scalar('Training/recall', recall,epoch)
        writer.add_scalar('Training/f1', f1,epoch)

        scheduler.step()

        # Store latest checkpoint in each epoch
        torch.save(
            {
                "model": model.state_dict(),
                "optim": optimizer.state_dict(),
                "scheduler":scheduler.state_dict(),
                "epoch": epoch,

            },
            os.path.join(args.output_dir, args.latest_checkpoint_file),
        )

        # Save checkpoint
        if epoch % args.epochs_per_checkpoint == 0:

            torch.save(
                {
                    "model": model.state_dict(),
                    "optim": optimizer.state_dict(),
                    "epoch": epoch,
                },
                os.path.join(args.output_dir, f"checkpoint_{epoch:04d}.pt"),
            )

    writer.close()

def test():

    parser = argparse.ArgumentParser(description='PyTorch Medical Segmentation Testing')
    parser = parse_training_args(parser)
    args, _ = parser.parse_known_args()

    args = parser.parse_args()

    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.enabled = args.cudnn_enabled
    torch.backends.cudnn.benchmark = args.cudnn_benchmark

    from data_function import MedData_test

    if hp.mode == '2d':
        # from models.two_d.alexnet import alexnet
        # model = alexnet(num_classes=2)

        # from models.two_d.densenet import densenet121
        # model = densenet121(num_class=2)

        from models.two_d.googlenet import googlenet
        model = googlenet(num_class=2)

        # from models.two_d.mobilenet import mobilenet
        # model = mobilenet(class_num=2)

        # from models.two_d.nasnet import nasnet
        # model = nasnet(class_num=2)

        # from models.two_d.resnet import resnet101
        # model = resnet101(num_classes=2)

        # from models.two_d.resnext import resnext101
        # model = resnext101(class_names=2)

        # from models.two_d.vggnet import vgg16_bn
        # model = vgg16_bn(num_class=2)

    elif hp.mode == '3d':

        from models.three_d.resnet3d import generate_model
        model = generate_model(18,n_input_channels=1,n_classes=2)

        from models.three_d.resnext3d import generate_model
        model = generate_model(50,n_input_channels=1,n_classes=2)

        from models.three_d.densenet3d import generate_model
        model = generate_model(121,n_input_channels=1,num_classes=2)

    model = torch.nn.DataParallel(model, device_ids=devicess,output_device=[1])

    print("load model:", args.ckpt)
    print(os.path.join(args.output_dir, args.latest_checkpoint_file))
    ckpt = torch.load(os.path.join(args.output_dir, args.latest_checkpoint_file), map_location=lambda storage, loc: storage)

    model.load_state_dict(ckpt["model"])

    model.cuda()

    test_dataset = MedData_test(source_test_0_dir,source_test_1_dir)
    test_loader = DataLoader(test_dataset.testing_set,
                            batch_size=args.batch,
                            shuffle=True,
                            pin_memory=True,
                            drop_last=True)

    model.eval()

    predicts = []
    gts = []
    for i, batch in enumerate(test_loader):

        x = batch['source']['data']
        y = batch['label']

        x = x.type(torch.FloatTensor).cuda()
        y = y.type(torch.LongTensor).cuda()

        if hp.mode == '2d':
            x = x.squeeze(-1)
            x = x[:,:1,:,:]

        outputs = model(x)

        outputs_logit = outputs.argmax(dim=1)

        predicts.append(outputs_logit.cpu().detach().numpy())
        gts.append(y.cpu().detach().numpy())

    predicts = np.concatenate(predicts).flatten().astype(np.int16)
    gts = np.concatenate(gts).flatten().astype(np.int16)
    acc = metrics.accuracy_score(predicts, gts)
    recall = metrics.recall_score(predicts, gts)
    f1 = metrics.f1_score(predicts, gts)
    ## log
    print("acc:"+str(acc))
    print("recall:"+str(recall))
    print("f1:"+str(f1))
    print(metrics.confusion_matrix(predicts, gts))

if __name__ == '__main__':
    if hp.train_or_test == 'train':
        train()
    elif hp.train_or_test == 'test':
        test()


import bitcoin
import base58
from bitcoin import privtopub, pubkey_to_address
from klefki.crypto.ecdsa.secp256k1 import pubkey, random_privkey
from klefki.bitcoin import gen_key_pair
from klefki.bitcoin.address import gen_address_from_priv
from klefki.bitcoin.public import decode_pubkey, gen_pub_key
from klefki.bitcoin.private import encode_privkey, decode_privkey

def test_priv_encode():
    key = random_privkey()
    encoded = encode_privkey(key)
    ans = bitcoin.encode_privkey(key.value, 'wif_compressed')
    t1 = base58.b58decode(ans)
    t2 = base58.b58decode(encoded)
    assert t1 == t2
    assert encoded == ans

def test_key():
    priv, pub = gen_key_pair()
    assert pub == privtopub(priv)
    priv, pub = gen_key_pair()
    assert pub == privtopub(priv)

    priv, pub = gen_key_pair()
    assert pub == privtopub(priv)

    priv, pub = gen_key_pair()
    assert pub == privtopub(priv)

    priv, pub = gen_key_pair()
    assert pub == privtopub(priv)

def test_addr():
    key = random_privkey()
    pub = gen_pub_key(key)
    addr = gen_address_from_priv(key)
    assert addr == pubkey_to_address(pub)

    key = random_privkey()
    pub = gen_pub_key(key)
    addr = gen_address_from_priv(key)
    assert addr == pubkey_to_address(pub)

def test_decode_pub():
    key = random_privkey()
    ans = pubkey(key)
    ret = decode_pubkey(gen_pub_key(key))
    assert ans == ret

    key = random_privkey()
    ans = pubkey(key)
    ret = decode_pubkey(gen_pub_key(key))
    assert ans == ret

def test_decode_priv():
    key = random_privkey()
    res = decode_privkey(encode_privkey(key))
    assert res == key


from ..utils import alloc_logger
from ..config.DefaultConfig import DefaultConfig
from .label_transformer import LabelTrasformer
from .label_formatter import LabelFormatter
from .label_file_reader import LabelFileReader
from .label_file_reader import LabelInfo
import os
import json
import re

class ReResultFormatter:
    def __init__(self, split_index_file: str = None, end='\n'):
        super().__init__()

        self.logger = alloc_logger(
            "re_result_formatter.log", ReResultFormatter)
        self.end = end
        email_pattern = r"[a-zA-Z0-9_\.]+@[a-zA-PR-Z0-9_\.]+"
        mobile_pattern = r"[\+＋]?([\(（]\d+[\)）])?\d[\-\d]*\d([转轉]\d+)?"
        QQ_pattern = r"\d{9,10}"
        date_pattern = r"([12]\d{3}\-\d{1,2}-\d{1,2})|([12]\d{3}\-[12]\d{3})"
        self.email_re = re.compile(email_pattern)
        self.mobile_re = re.compile(mobile_pattern)
        self.QQ_re = re.compile(QQ_pattern)
        self.date_re = re.compile(date_pattern)
        self.combine_index = {}  # origin: (beg, target)
        split_index_file = split_index_file if split_index_file is not None else os.path.join(
            DefaultConfig.PATHS.DATA_INFO, "split_index_test.json")
        self.logger.log_message(
            "loading split_index from file:\t", split_index_file)

        with open(split_index_file, 'r', encoding='utf8') as f:
            m = json.load(f)
            for item in m:
                target = item["target"]
                origin = item["origin"]
                beg = item["beg"]
                if origin in self.combine_index.keys():
                    self.combine_index[origin].append((beg, target))
                else:
                    self.combine_index[origin] = [(beg, target)]

    def combine_all(self, receive_rate: float = 0.8, result_dir: str = None):
        signature = "conbine_all()\t"

        self.logger.log_message(signature, "start!")

        result_dir = result_dir if result_dir is not None else os.path.join(
            DefaultConfig.PATHS.DATA_CCF_CLEANED, "test/label")
        self.logger.log_message(signature, "origin result dir:\t", result_dir)

        def file_num_to_tokens_and_labels(num: int) -> (list, list):
            with open(os.path.join(result_dir, "{:d}.json".format(num)), 'r', encoding='utf8') as f:
                result = json.load(f)
                return result["data"][1:-1], result["tags"][1:-1]

        origin_result_count = len(os.listdir(
            os.path.join(DefaultConfig.PATHS.DATA_CCF_RAW, "test")))
        self.logger.log_message(
            signature, "origin result count:\t", origin_result_count)

        label_formatter = LabelFormatter()
        label_formatter.load_transformer_from_file()

        reader = LabelFileReader()

        output_csv = open(os.path.join(
            DefaultConfig.PATHS.DATA_INFO, "predict_origin.csv"), 'w', encoding='utf8')

        label_token_len_mismatch = []
        for i in range(origin_result_count):
            # labels = None
            tokens, labels = file_num_to_tokens_and_labels(i)
            if i in self.combine_index.keys():
                targets = self.combine_index[i]
                targets.sort(key=lambda t: t[0])
                for _, target in targets:
                    new_tokens, new_labels = file_num_to_tokens_and_labels(
                        target)
                    labels += new_labels
                    tokens += new_tokens
            if len(tokens) != len(labels):
                label_token_len_mismatch.append(i)
                continue
            infos = label_formatter.bio_str_list_label_and_token_list_to_infos(
                ID=i, bio_str_list=labels, tokens=tokens, receive_rate=receive_rate)
            for info in infos:
                string = reader.dumps(info)
                output_csv.write(string + self.end)
        output_csv.close()

        self.logger.log_message("mismatches:\t", label_token_len_mismatch)

    def detect_email(self, ID: int, data: str) -> "List[LabelInfo]":
        signature = "detect_email()\t"
        ret = []
        for m in self.email_re.finditer(data):
            ret.append(LabelInfo(
                ID=ID,
                Category="email",
                Pos_b=m.start(),
                Pos_e=m.end() - 1,
                Privacy=m.group()
            ))
        # if len(ret) != 0:
        #     self.logger.log_message(signature, "[{:d}] found {:d} emails by re".format(ID, len(ret)))
        #     self.logger.log_message(ret)
        return ret

    def detect_mobile_and_QQ(self, ID: int, data: str) -> "List[LabelInfo]":
        signature = "detect_mobile_and_QQ()\t"
        ret = []
        for m in self.mobile_re.finditer(data):
            content = m.group()
            if len(content) < 8:
                continue
            count = 0
            num_list = [str(i) for i in range(10)]
            for ch in content:
                if ch in num_list:
                    count += 1
            if count > 15:
                continue
            if self.date_re.fullmatch(content):
                continue
            clz = "mobile"
            if self.QQ_re.fullmatch(content):
                clz = "QQ"
            ret.append(LabelInfo(
                ID=ID,
                Category=clz,
                Pos_b=m.start(),
                Pos_e=m.end() - 1,
                Privacy=m.group()
            ))
        # if len(ret) != 0:
        #     self.logger.log_message(signature, "[{:d}] found {:d} QQ/mobile by re".format(ID, len(ret)))
        #     self.logger.log_message(ret)
        return ret

    def _print_infos_to_csv_for_id(self, ID: int, infos: list, csv_ofs, data_dir: str, detect_email=True) -> (int, int, int):
        with open(os.path.join(data_dir, "{:d}.txt".format(ID)), 'r', encoding='utf8') as f:
            raw_content = f.read()
        reader = LabelFileReader()
        signature = "_print_infos_to_csv_for_id()\t"
        not_in_raw_count = 0
        head = 0
        to_print_infos = []
        for info in infos:
            content = info.Privacy.replace(',', '').replace('\n', '')

            current_content = raw_content[head:].lower()
            if content in current_content:
                new_beg = head + current_content.find(content)
                new_end = new_beg + len(content) - 1
                head = new_end + 1
                to_print_infos.append(LabelInfo(
                    ID=ID,
                    Category=info.Category,
                    Pos_b=new_beg,
                    Pos_e=new_end,
                    Privacy=raw_content[new_beg: new_end+1]
                ))
                continue
            pattern = content               \
                .replace('\\', '\\\\')      \
                .replace('(', '\\(')        \
                .replace(')', '\\)')        \
                .replace('?', '\\?')        \
                .replace('.', '\\.')        \
                .replace('*', '\\*')        \
                .replace('+', '\\+')        \
                .replace("[UNK]", ".+?")    \
                .replace('[', '\\[')        \
                .replace(']', '\\]')        \
                .replace('{', '\\{')        \
                .replace('}', '\\}')
            m = re.search(pattern, current_content)
            if m is not None:
                new_beg = head + m.start()
                new_content = m.group()
                new_end = new_beg + len(new_content) - 1
                head = new_end + 1
                to_print_infos.append(LabelInfo(
                    ID=ID,
                    Category=info.Category,
                    Pos_b=new_beg,
                    Pos_e=new_end,
                    Privacy=raw_content[new_beg: new_end+1]
                ))
                continue
            not_in_raw_count += 1
            if not_in_raw_count == 1:
                self.logger.log_message(
                    signature, "[{:d}] found mismatch(es):".format(ID))
            self.logger.log_message(signature, "pattern=", pattern)
            self.logger.log_message(signature, '\t', reader.dumps(info))

        for info in to_print_infos:
            csv_ofs.write(reader.dumps(info) + '\n')
        email_count = 0
        mobile_QQ_count = 0
        if detect_email:
            for info in self.detect_email(ID, raw_content):
                email_count += 1
                csv_ofs.write(reader.dumps(info) + '\n')
            for info in self.detect_mobile_and_QQ(ID, raw_content):
                mobile_QQ_count += 1
                csv_ofs.write(reader.dumps(info) + '\n')

        if not_in_raw_count != 0:
            self.logger.log_message(
                signature, "[{:d}] detect {:d} info not in raw".format(ID, not_in_raw_count))

        return not_in_raw_count, email_count, mobile_QQ_count

    def trans_origin_to_raw(self, data_dir: str = None, detect_email=True):
        signature = "trans_origin_to_raw()\t"

        input_csv = open(os.path.join(DefaultConfig.PATHS.DATA_INFO,
                                      "predict_origin.csv"), 'r', encoding='utf8')
        output_csv = open(os.path.join(
            DefaultConfig.PATHS.DATA_INFO, "predict_raw.csv"), 'w', encoding='utf8')

        reader = LabelFileReader()

        data_dir = data_dir if data_dir is not None else os.path.join(
            DefaultConfig.PATHS.DATA_CCF_RAW, "test")

        not_in_raw_count = 0
        email_count = 0
        mobile_QQ_count = 0
        current_id = 0
        infos = []
        for line in input_csv.readlines():
            info = reader.loads(line)
            if (info.ID == current_id):
                infos.append(info)
                continue
            not_in_raw_count_delta, email_count_delta, mobile_QQ_count_delta =  \
                self._print_infos_to_csv_for_id(
                    ID=current_id,
                    infos=infos,
                    csv_ofs=output_csv,
                    data_dir=data_dir,
                    detect_email=detect_email)
            not_in_raw_count += not_in_raw_count_delta
            email_count += email_count_delta
            mobile_QQ_count += mobile_QQ_count_delta

            current_id = info.ID
            infos = [info]

        not_in_raw_count_delta, email_count_delta, mobile_QQ_count_delta =  \
            self._print_infos_to_csv_for_id(
                ID=current_id,
                infos=infos,
                csv_ofs=output_csv,
                data_dir=data_dir,
                detect_email=detect_email)
        not_in_raw_count += not_in_raw_count_delta
        email_count += email_count_delta
        mobile_QQ_count += mobile_QQ_count_delta

        self.logger.log_message(
            signature, "not in raw count=", not_in_raw_count)
        self.logger.log_message(signature, "reg email count=", email_count)
        self.logger.log_message(signature, "reg mobile&QQ count=", mobile_QQ_count)
        self.logger.log_message(signature, "finish")
        input_csv.close()
        output_csv.close()

    def final_format(self):
        signature = "final_format()\t"
        self.logger.log_message(signature, "start")
        with open(os.path.join(DefaultConfig.PATHS.DATA_INFO, "predict_raw.csv"), 'r', encoding='utf8') as f:
            lines = f.read().splitlines()
            self.logger.log_message(
                signature, "total entity count=", len(lines))
            all_content = list(set(lines))
            self.logger.log_message(
                signature, "unique entity count=", len(all_content))

        def get_beg(line: str) -> int:
            ID, clz, beg, end, content = tuple(line.split(','))
            return int(beg)

        def get_end(line: str) -> int:
            ID, clz, beg, end, content = tuple(line.split(','))
            return int(end)

        def get_id(line: str) -> int:
            ID, clz, beg, end, content = tuple(line.split(','))
            return int(ID)

        def is_single(line: str) -> int:
            ID, clz, beg, end, content = tuple(line.split(','))
            return int(beg) == int(end)

        def is_mobile(line: str) -> int:
            ID, clz, beg, end, content = tuple(line.split(','))
            return clz == "mobile"

        def is_QQ(line: str) -> int:
            ID, clz, beg, end, content = tuple(line.split(','))
            return clz == "QQ"

        def get_content(line: str) -> int:
            ID, clz, beg, end, content = tuple(line.split(','))
            return content
        all_content.sort(key=get_end)
        all_content.sort(key=get_beg)
        all_content.sort(key=get_id)
        self.logger.log_message(signature, "len=0:")
        self.logger.log_message(signature, "length-1 entity count=",
                                sum(1 for line in all_content if is_single(line)))
        # for line in all_content:
        #     if (is_mobile(line)):
        #         self.logger.log_message(
        #             signature, "[mobiles]\t", get_content(line))
        # for line in all_content:
        #     if (is_QQ(line)):
        #         self.logger.log_message(signature, "[QQ]\t", get_content(line))
        with open(os.path.join(DefaultConfig.PATHS.DATA_INFO, "predict.csv"), 'w', encoding='utf8') as f:
            f.write('ID,Category,Pos_b,Pos_e,Privacy\n')
            for content in all_content:
                if not is_single(content):
                    f.write(content + "\n")

if __name__ == "__main__":
    # formatter = ReResultFormatter(os.path.join(DefaultConfig.PATHS.DATA_INFO, "split_index_train.json"))
    # formatter.combine_all(
    #    origin_data_count=2515,
    #    label_dir=os.path.join(DefaultConfig.PATHS.DATA_CCF_CLEANED, "train/label"),
    #    data_dir=os.path.join(DefaultConfig.PATHS.DATA_CCF_CLEANED, "train/data"))
    # formatter.trans_origin_to_raw(data_dir=os.path.join(DefaultConfig.PATHS.DATA_CCF_RAW, "train/data"))
    formatter = ReResultFormatter()
    formatter.combine_all(receive_rate = 0.7, result_dir=os.path.join(
        DefaultConfig.PATHS.DATA_CCF_CLEANED, "test/label/80/30"))
    formatter.trans_origin_to_raw()
    formatter.final_format()


from urllib.request import urlopen
from bs4 import BeautifulSoup
#import ssl
import re

# Ignore SSL certificate errors
#ctx = ssl.create_default_context()
#ctx.check_hostname = False
#ctx.verify_mode = ssl.CERT_NONE

url = input('Enter - ')
html = urlopen(url).read()
soup = BeautifulSoup(html, "html.parser")

# Retrieve all of the span tags
tags = soup('span')
total = 0
for tag in tags:
    # Look at the parts of a tag
    content = re.findall('[0-9]+', tag.contents[0])
    total += int(content[0])
print(total)


import numpy
import cv2
import time
from package.image import CS_IIP, CS_BGR, CS_HSV, iip_max, iip_min, hsvmax, colorspace_name
from sklearn.externals.joblib import Parallel, delayed
import numpy as np

__author__ = 'luigolas'

def _parallel_transform(fe, *args):
    return fe.extract(*args)

class FeatureExtractor(object):
    def extract_dataset(self, dataset, n_jobs, verbosity=2, calc4train_set=False):
        """

        :param dataset:
        :param n_jobs:
        :param verbosity:
        :param calc4train_set:
        :return:
        """
        raise NotImplementedError("Please Implement extract_dataset method")

    def extract(self, image, *args):
        """

        :param image:
        :param mask:
        :raise NotImplementedError:
        """
        raise NotImplementedError("Please Implement evaluate method")

    def dict_name(self):
        """

        :return:
        """
        raise NotImplementedError("Please Implement dict_name method")

class Histogram(FeatureExtractor):
    color_ranges = {CS_BGR: [0, 256, 0, 256, 0, 256],
                    CS_IIP: [80.86236, 140.3336, -17.9941, 18.1964, -73.6702, -42.93588],
                    # CS_IIP: [iip_min, iip_max] * 3,
                    CS_HSV: [0, hsvmax[0], 0, hsvmax[1], 0, hsvmax[2]]}

    # color_channels = {CS_BGR: [0, 1, 2],
    #             CS_IIP: [0, 1, 2],
    #             CS_HSV: [0, 1, 2]}

    def __init__(self, color_space, bins=None, dimension="1D"):
        if color_space is None or not isinstance(color_space, int):
            raise AttributeError("Colorspace parameter must be valid")
        self._colorspace = color_space
        if bins is None:
            bins = 32
        if type(bins) == int:
            if color_space == CS_BGR:
                bins = [bins] * 3
            elif color_space == CS_HSV:
                bins = [bins] * 2
            elif color_space == CS_IIP:
                bins = [bins] * 3

        self._channels = [i for i, e in enumerate(bins) if e != 0]
        self._original_bins = bins
        self._bins = [i for i in bins if i != 0]

        if not isinstance(dimension, str) or not (dimension == "1D" or dimension == "3D"):
            raise AttributeError("Dimension must be \"1D\" or \"3D\"")
        self._dimension = dimension

        self.name = "Histogram_%s_%s_%s" % (colorspace_name[self._colorspace], self._bins, self._dimension)

    def extract_dataset(self, dataset, n_jobs=-1, verbosity=2, calc4train_set=False):
        if verbosity > 1: print("   Calculating Histograms %s, %s" % (colorspace_name[self._colorspace],
                                                                      str(self._original_bins)))
        if calc4train_set:
            images = dataset.probe.images_train + dataset.probe.images_test
            images += dataset.gallery.images_train + dataset.gallery.images_test
        else:
            images = dataset.probe.images_test
            images += dataset.gallery.images_test

        if dataset.probe.masks_test:
            if calc4train_set:
                masks = dataset.probe.masks_train + dataset.probe.masks_test
                masks += dataset.gallery.masks_train + dataset.gallery.masks_test
            else:
                masks = dataset.probe.masks_test
                masks += dataset.gallery.masks_test
        else:
            masks = [None] * (len(images))

        if dataset.probe.regions_test:
            if calc4train_set:
                regions = dataset.probe.regions_train + dataset.probe.regions_test
                regions += dataset.gallery.regions_train + dataset.gallery.regions_test
            else:
                regions = dataset.probe.regions_test
                regions += dataset.gallery.regions_test
        else:
            regions = [None] * (len(images))

        if dataset.probe.maps_test:
            if calc4train_set:
                maps = dataset.probe.maps_train + dataset.probe.maps_test
                maps += dataset.gallery.maps_train + dataset.galley.maps_test
            else:
                maps = dataset.probe.maps_test
                maps += dataset.gallery.maps_test
        else:
            maps = [None] * (len(images))

        args = ((im, mask, region, m) for im, mask, region, m in zip(images, masks, regions, maps))

        results = Parallel(n_jobs)(delayed(_parallel_transform)(self, im, mask, reg, m) for im, mask, reg, m in args)

        test_len = dataset.test_size
        if calc4train_set:
            train_len = dataset.train_size
            dataset.probe.fe_train = np.asarray(results[:train_len])
            dataset.probe.fe_test = np.asarray(results[train_len:train_len + test_len])
            dataset.gallery.fe_train = np.asarray(results[train_len + test_len:-test_len])
            dataset.gallery.fe_test = np.asarray(results[-test_len:])
        else:
            dataset.probe.fe_test = np.asarray(results[:test_len])
            dataset.gallery.fe_test = np.asarray(results[-test_len:])

    def extract(self, image, mask=None, regions=None, weights=None, normalization=cv2.NORM_MINMAX):
        """

        :param image:
        :param mask:
        :param regions:
        :param weights:
        :param normalization:
        :return:
        """

        if self._colorspace and self._colorspace != image.colorspace:
            # image = image.to_color_space(self._colorspace, normed=True)
            image = image.to_color_space(self._colorspace)

        if regions is None:
            regions = [[0, image.shape[0], 0, image.shape[1]]]

        histograms = []
        for region in regions:
            im = image[region[0]:region[1], region[2]:region[3]]

            if mask is not None and len(mask):
                m = mask[region[0]:region[1], region[2]:region[3]]
            else:
                m = mask

            if weights is not None and len(weights):
                weight = weights[region[0]:region[1], region[2]:region[3]]
            else:
                weight = 1

            hist = self.calcHistNormalized(im, m, weight, normalization)
            histograms.append(hist)

        return numpy.asarray(histograms).flatten().clip(0)

    # noinspection PyPep8Naming
    def calcHistNormalized(self, image, mask, weight, normalization):
        """

        :param image:
        :param mask:
        :param weight:
        :param normalization:
        :return:
        """
        # Using mask and map
        if mask is not None and len(mask):
            mask = mask * weight
        elif type(weight) != int:
            mask = weight
        else:
            mask = None

        # mask = weight                       # Using only MAP

        # mask = np.ones_like(mask)             # Not using mask nor map

        #  All comented, use only mask

        if self._dimension == "3D":
            # TODO add weighted ?
            # _channels = list(range(len(self._bins)))
            channels = numpy.nonzero(self._bins)

            hist = cv2.calcHist([image], channels,
                                mask, self._bins, Histogram.color_ranges[image.colorspace])
            # hist = self.normalize_hist(hist, normalization)
        else:  # 1D case
            hist = []
            for channel, bins in zip(self._channels, self._bins):
                if type(weight) != int:
                    h = calc_hist(image, channel, mask, bins,
                                  Histogram.color_ranges[image.colorspace][channel*2:channel*2+2])
                                  # [0., 1.])

                else:
                    h = cv2.calcHist([image], [channel], mask, [bins],
                                     Histogram.color_ranges[image.colorspace][channel*2:channel*2+2])
                                     # [0., 1.])

                # hist.extend(self.normalize_hist(h.astype(np.float32), normalization))
                hist.extend(h.astype(np.float32))
                # In 1D case it can't be converted to numpy array as it might have different dimension (bins) sizes
        return hist

    @staticmethod
    def normalize_hist(histogram, normalization=cv2.NORM_MINMAX):
        """
        if histogram.max() != 0:  # Some cases when mask completely occlude region

        cv2.normalize(histogram, histogram, 0, 1, cv2.NORM_MINMAX, -1)  # Default
        cv2.normalize(histogram, histogram, 1, 0, cv2.NORM_L1)  # Divide all by "num pixel". Sum of all values equal 1
        cv2.normalize(histogram, histogram, 1, 0, cv2.NORM_INF)  # Divide all values by max

        Any of the three have the same effect. Will use NORM_MINMAX
        :param histogram:
        :param normalization: if None, no nomalization takes effect
        :return:
        """
        if normalization == cv2.NORM_MINMAX:
            cv2.normalize(histogram, histogram, 0, 1, cv2.NORM_MINMAX, -1)  # Default
        elif (normalization == cv2.NORM_L1) or (normalization == cv2.NORM_INF):
            cv2.normalize(histogram, histogram, 1, 0, normalization)

        return histogram

    def dict_name(self):
        return {"Feature_Extractor": "Histogram", "FeColorSpace": colorspace_name[self._colorspace],
                "FeBins": str(self._original_bins), "FeDim": self._dimension}

def calc_hist(im, ch, weight, bins, hist_range):
    """
    Inspiration: https://gist.github.com/nkeim/4455635
    Fast 1D histogram calculation. Must have all values initialized
    :param im: image to calculate histogram. Must be os shape (Height, Width, _channels)
    :param ch: channel to calculate histogram
    :param weight: weight map for values on histogram. Must be of shape (Height, Width) and float type
    :param bins: Numbers of bins for histogram
    :param hist_range: Range of values on selected channel
    :return:
    """
    # return np.bincount(np.digitize(im[:, :, ch].flatten(), np.linspace(hist_range[0], hist_range[1], bins + 1)[1:-1]),
    #                    weights=weight.flatten(), minlength=bins)
    return np.bincount(np.digitize(im[:, :, ch].flatten(), np.linspace(hist_range[0], hist_range[1], bins)[1:]),
                       weights=weight.flatten(), minlength=bins)


#-*- coding:utf-8 -*-

import cv2
import os
import time
import numpy as np
import matplotlib.pyplot as plt
import imutils

'''
1. 辨识BIOS光标位置  两个方案：1. 颜色识别，2. 轮廓识别
'''

picPath = "C:\\Users\\Laurence_SZH\\Pictures\\Camera Roll\\201124183637.BMP"
fullPath = os.path.expanduser(picPath)
print(fullPath)

pic = cv2.imread(fullPath)
print(pic)
#pic2 = pic[:,:,[2,1,0]]
pic2 = cv2.cvtColor(pic, cv2.COLOR_BGR2RGB)
#plt.subplot(20, 20, 10)
#以下这段主要是show出我们读到的图片

#用plt.show的好处是可以读得到threshod的值
plt.imshow(pic2)
plt.title('BGR')
#plt.show()  # 用这个会一直显示
plt.ion()  #搭配plt.pause & plt.close()可以让窗口显示5秒就关闭
plt.pause(5)
#plt.waitforbuttonpress(4)
plt.close()  #关闭图像窗口

# 利用阀值_绘制长方形的BIOS光标轮廓
img_gray = cv2.cvtColor(pic, cv2.COLOR_BGR2GRAY)
# ret, thresh = cv2.threshold(img_gray, 16, 18, 0)
ret, thresh = cv2.threshold(img_gray, 33, 0, 0)
plt.imshow(img_gray)
plt.title('thresh')
plt.ion()
plt.pause(35)
plt.close()

#检测图像的连通区(输入为二值化图像)
threshold, contours, heirarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
# 绘制寻找到的轮廓线
img_contours = cv2.drawContours(pic, contours, -1, (255,255,255),3)

cv2.imshow('contours', img_contours)
cv2.waitKey(0) & 0xFF == ord('q')

# 测试一下那个阀值的参数更适合我们的BIOS介面
#import numpy as np
#from matplotlib import pyplot as plt

#img = cv2.imread(r'/Users/Documents/image01.jpg',0)   # 读入灰度图
ret,thresh1 = cv2.threshold(img_gray,127,255,cv2.THRESH_BINARY)
ret,thresh2 = cv2.threshold(img_gray,127,255,cv2.THRESH_BINARY_INV)
ret,thresh3 = cv2.threshold(img_gray,127,255,cv2.THRESH_TRUNC)
ret,thresh4 = cv2.threshold(img_gray,127,255,cv2.THRESH_TOZERO)
ret,thresh5 = cv2.threshold(img_gray,127,255,cv2.THRESH_TOZERO_INV)

titles = ['Original Image','BINARY','BINARY_INV','TRUNC','TOZERO','TOZERO_INV']
images = [img_gray, thresh1, thresh2, thresh3, thresh4, thresh5]

for i in range(6):
    plt.subplot(2,3,i+1),plt.imshow(images[i],'gray')
    plt.title(titles[i])
    plt.xticks([]),plt.yticks([])

plt.show()


from ast import Store
from service.db_service import get_db_values, write_to_db
from store.user_store import UserStore
from utils import Transactions

def show_transaction_options():
    user = UserStore.get_user()
    while True:
        print("Login Successful")
        print("Transaction Options")
        print("1. ATM Withdrawal")
        print("2. Deposit")
        print("3. Transfer")
        print("4. Show All transactions")
        print("5. Show Top Customers")
        print("0. Quit")
        option = int(input("Kindly enter option: "))
        if option == 1:
            withdraw(user)
        elif option == 2:
            deposit(user)
        elif option == 3:
            transfer(user)
        elif option == 4:
            show_all_transactions(user)
        elif option == 5:
            show_top_customers()
        elif option == 0:
            exit()

def show_top_customers():
    no_of_customers = int(input("Enter how many number of top customers to fetch"))
    db_values = get_db_values()
    customers = db_values["users"]["data"]
    customers = sorted(customers, key=lambda d: d["balance"], reverse=True)[
        0:no_of_customers
    ]
    count = 1
    for customer in customers:
        print("Top Customers")
        print(f"{count} Name: {customer['name']}")
        print("-------------------------------------")
        count += 1

def withdraw(user, amount=None, transaction=True):
    if amount is None:
        amount = int(input("Enter the Amount to withdraw "))

    debit_amount = user["balance"] - amount
    if debit_amount < 1000:
        print("Insuffecient funds")
        return False
    user["balance"] = debit_amount
    print(debit_amount)
    print(user["balance"])

    db_values = get_db_values()
    for _user in db_values["users"]["data"]:
        if _user["customer_id"] == user["customer_id"]:
            _user["balance"] = user["balance"]
    if transaction:
        t = Transactions.WITHDRAW
        db_values = store_transaction(db_values, t.value, user, amount)
    print(db_values["users"]["data"])
    write_to_db(db_values)
    return True

def deposit(user, amount=None, transaction=True):
    if amount is None:
        amount = int(input("Enter the Amount to deposit "))

    credit_amount = user["balance"] + amount
    user["balance"] = credit_amount
    db_values = get_db_values()
    for _user in db_values["users"]["data"]:
        if _user["customer_id"] == user["customer_id"]:
            _user["balance"] = user["balance"]
    if transaction:
        t = Transactions.DEPOSIT
        db_values = store_transaction(db_values, t.value, user, amount)

    write_to_db(db_values)

def transfer(user):
    while True:
        account_number = input("Input Account number to transfer: ")
        if account_number == user["account_number"]:
            print("Cannot transfer to your own account")
            continue
        amount = int(input("Amount to transfer"))
        benefeciary = None
        db_values = get_db_values()
        for _user in db_values["users"]["data"]:
            if _user["account_number"] == account_number:
                benefeciary = _user
                break
        if benefeciary == None:
            print("Benefeciary not found. Try again")
            continue
        break

    can_withdraw = withdraw(user, amount, False)
    if not can_withdraw:
        print("Insuffecient funds")
        return
    deposit(benefeciary, amount, False)
    t = Transactions.TRANSFER
    db_values = get_db_values()
    db_values = store_transaction(db_values, t.value, user, amount)

    write_to_db(db_values)

def store_transaction(values, type, user, amount):
    last_id = values["transactions"]["last_transaction_id"]
    transaction_obj = {
        "id": last_id + 1,
        "type": type,
        "user_id": user["customer_id"],
        "amount": amount,
        "balance": user["balance"],
    }
    values["transactions"]["data"].append(transaction_obj)
    return values

def show_all_transactions(user):
    db_values = get_db_values()
    transactions = []
    for transaction in db_values["transactions"]["data"]:
        if transaction["user_id"] == user["customer_id"]:
            transactions.append(transaction)

    for transaction in transactions:
        print("Transactions")
        print(f"ID: {transaction['id']}")
        print(f"Type: {transaction['type']}")
        print(f"Amount: {transaction['amount']}")
        print(f"Balance: {transaction['balance']}")
        print("-------------------------------------")


import scrapy
# to scrape pages:
# python3 scrapy crawl prices -o prices.json

class PriceSpider(scrapy.Spider):
    name = 'prices'

    # Change query variable to search for different items
    def start_requests(self):
        query = 'iphone'
        urls = [
            f'https://newyork.craigslist.org/search/sss?query={query}&sort=rel'
        ]

        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        for item in response.css('li.result-row'):
            yield {
                'title': item.css('.result-title::text')[0].get(),
                'price': item.css('.result-price::text')[0].get(),
                'link': item.css('.result-title::attr(href)').get()
            }

        next_page = response.css('a.button.next::attr(href)').get()
        if next_page is not None:
            next_page = response.urljoin(next_page)
            yield scrapy.Request(next_page, callback=self.parse)


from db.database import db
from models.models import Card, Deck, Rating, User,DeckStat,Participation,Performance
from sqlalchemy import *
import random
import hashlib
from datetime import datetime
import os
import time
#import matplotlib.pyplot as plt
#import base64
#from io import BytesIO

def sha3512(password):
    m = hashlib.sha3_512()
    m.update(password.encode("utf-8"))
    return str(m.hexdigest())

def username_exists(username):
    user = db.session.query(User).filter(User.username==username).first()
    if user == None:
        return False
    else:
        return True

def email_exists(email):
    user = db.session.query(User).filter(User.email==email).first()
    if user == None:
        return False
    else:
        return True

def get_user_by_username(username):
    user = db.session.query(User).filter(User.username==username).first()
    return user

def get_user_by_id(user_id):
    user = db.session.query(User).filter(User.id==user_id).first()
    return user

def get_deck_by_deck_id(deck_id):
    deck = db.session.query(Deck).filter(Deck.deck_id==deck_id).first()
    return deck
def get_deckstat(user_id,deck_id):
    stat = db.session.query(DeckStat).filter(DeckStat.deck_id==deck_id,DeckStat.user_id==user_id).first()
    return stat

def get_decks_for_dashboard(user_id):
    deckstats = db.session.query(DeckStat).filter(DeckStat.user_id==user_id).all()
    deck_list = []
    for deckstat in deckstats:
        number_of_cards = countcards(deckstat.deck_id)
        deck = get_deck_by_deck_id(deckstat.deck_id)
        owner = get_user_by_id(deck.owner).username
        visibility = deck.visibility
        name = deck.name
        deck_id=deck.deck_id
        description = deck.description
        average_score = deckstat.average_score
        rating=get_rating(deckstat.deck_id)
        times_reviewed = deckstat.times_reviewed
        last_reviewed = format_datetime(deckstat.last_reviewed)
        deck_obj = {'owner':owner,
        'visibility':visibility,
        'name':name,
        'description':description,
        'average_score':average_score,
        'rating':rating,
        'times_reviewed':times_reviewed,
        'last_reviewed':last_reviewed,
        'deck_id':deck_id,
        'number_of_cards':number_of_cards}
        if str(deck.owner) == str(user_id) or deck.visibility == "Public":
            deck_list.append(deck_obj)
    return deck_list

def countcards(deck_id):
    cards = db.session.query(Card).filter(Card.deck_id==deck_id).all()
    return len(cards)

def get_decks_for_user(user_id,purpose):
    if purpose == 'all':
        decks = db.session.query(Deck).filter(or_(Deck.owner==user_id,Deck.visibility=='Public')).all()
    elif purpose == 'restricted':
        decks = db.session.query(Deck).filter(Deck.owner==user_id).all()
    deck_list = []
    for deck in decks:
        deckstat = db.session.query(DeckStat).filter(DeckStat.deck_id==deck.deck_id).filter(DeckStat.user_id==user_id).first()
        if deckstat==None:
            last_reviewed='Never'
        else:
            last_reviewed=format_datetime(deckstat.last_reviewed)
        deck_id = deck.deck_id
        number_of_cards = countcards(deck.deck_id)
        owner = get_user_by_id(deck.owner).username
        visibility = deck.visibility
        name = deck.name
        description = deck.description
        deck_obj = {'owner':owner,
        'visibility':visibility,
        'name':name,
        'description':description,
        'number_of_cards':number_of_cards,
        'deck_id':deck_id,
        'last_reviewed':last_reviewed}
        deck_list.append(deck_obj)
    return deck_list

def get_cards_by_deck(deck_id):
    deck = db.session.query(Deck).filter(Deck.deck_id==deck_id).first()
    cards = db.session.query(Card).filter(Card.deck_id==deck_id).all()
    card_list = []
    deck_name = deck.name
    for card in cards:
        card_id = card.card_id
        question = card.question
        hint = card.hint
        answer = card.answer
        card_obj = {'deck_id':deck_id,
        'card_id':card_id,
        'question':question,
        'hint':hint,
        'answer':answer,
        'deck_name':deck_name
        }
        card_list.append(card_obj)
    return card_list

def add_card(card):
    cards = db.session.add(card)
    db.session.commit()

def delete_card(card_id):
    db.session.query(Card).filter(Card.card_id==card_id).delete()
    db.session.commit()

def add_deck(deck):
    db.session.add(deck)
    db.session.commit()

def update_deck(deck_id,deck_name,deck_description,deck_visibility):
    db.session.query(Deck).filter(Deck.deck_id==deck_id).update({"name":deck_name,"description":deck_description,'visibility':deck_visibility})
    db.session.commit()

def delete_deck(deck_id):
    delete_rating(deck_id)
    delete_deckstats(deck_id)
    for card in get_cards_by_deck(deck_id):
        delete_card(card['card_id'])
    db.session.query(Deck).filter(Deck.deck_id==deck_id).delete()
    db.session.commit()

def check_if_deck_owner(deck_id,user_id):
    deck = db.session.query(Deck).filter(Deck.deck_id==deck_id).first()
    if deck:
        print(deck.owner,user_id)
        if str(deck.owner) == str(user_id):
            return True,deck.name,deck.description,deck.visibility
        else:
            return False,deck.name,deck.description,deck.visibility
    else:
        return False,None

def load_decks_quiz_selector(user_id):
    deck_list = []
    temp_decks = db.session.query(Deck).all()
    for deck in temp_decks:
        count = countcards(deck.deck_id)
        if count > 0:
            deckstat = db.session.query(DeckStat).filter(DeckStat.deck_id==deck.deck_id).filter(DeckStat.user_id==user_id).first()
            if deckstat==None:
                last_reviewed='Never'
            else:
                last_reviewed=format_datetime(deckstat.last_reviewed)
            deck_id = deck.deck_id
            number_of_cards = count
            owner = get_user_by_id(deck.owner).username
            visibility = deck.visibility
            name = deck.name
            description = deck.description
            deck_obj = {'owner':owner,
            'visibility':visibility,
            'name':name,
            'description':description,
            'number_of_cards':number_of_cards,
            'deck_id':deck_id,
            'last_reviewed':last_reviewed}
            #print("deck is owned : ",str(deck.owner)==str(user_id))
            #print(deck.owner)
            #print(user_id)
            #print("deck is public : ",deck.visibility=='Public')
            if str(deck.owner) == str(user_id) or deck.visibility == 'Public':
                deck_list.append(deck_obj)
    return deck_list

def format_datetime(unix):
    return datetime.utcfromtimestamp(int(unix)).strftime('%Y-%m-%d %H:%M:%S')

def question_gen(deck_id):
    cards = db.session.query(Card).filter(Card.deck_id==deck_id).all()
    #deck = db.session.query(Deck).filter(Deck.deck_id==deck_id).first()
    questions_set = []
    with open('data/wordlist.txt') as f:
        W = f.readlines()
        n = len(W)
        index = 1
        for card in cards:
            options = [card.answer]
            while len(options) < 4:
                idx = random.randint(0,n-1)
                print(idx)
                if W[idx] not in options and W[idx]:
                    options.append(W[idx].capitalize())
            random.shuffle(options)
            question_obj = {
            'card_id':card.card_id,
            'deck_id':card.deck_id,
            'index':index,
            'question':card.question,
            'hint':card.hint,
            'options':options}
            questions_set.append(question_obj)
    return questions_set
            #print("options generated : ",options)

def export_deck(user_id,deck_id,endpoint):
    cards = db.session.query(Card).filter(Card.deck_id==deck_id).all()
    if len(cards) > 0:
        lines = []
        for card in cards:
            question = card.question
            hint = card.hint
            answer = card.answer
            lines.append("{},{},{}\n".format(question,hint,answer))
        with open("proc/{}.csv".format(endpoint), 'a+') as f:
            for line in lines:
                f.write(line)

def quick_quiz_score(deck_id,submission):
    cards = db.session.query(Card).filter(Card.deck_id==deck_id).all()
    number_of_cards = len(cards)
    correct_cards = 0
    for sub in submission:
        card_id = sub["card_id"]
        reply = sub["reply"]
        card = [card for card in cards if card.card_id == card_id][0]
        if card.answer == reply:
            correct_cards+=1
    return correct_cards,number_of_cards

def update_rating(deck_id,rating):
    rating_dict={"Easy":"0","Medium":"1","Hard":"2"}
    rating_val = rating_dict[rating]
    rating_obj = Rating(deck_id=deck_id,rating=rating_val)
    db.session.add(rating_obj)
    db.session.commit()

def delete_rating(deck_id):
    db.session.query(Rating).filter(Rating.deck_id==deck_id).delete()
    db.session.commit()

def get_rating(deck_id):
    ratings = db.session.query(Rating).filter(Rating.deck_id==deck_id).all()
    if len(ratings) == 0:
        return "No rating yet"
    rating_val = sum([int(rating.rating) for rating in ratings])/len(ratings)
    if rating_val<0.7:
        rating_string="Easy"
    elif rating_val<1.4:
        rating_string="Medium"
    else:
        rating_string="Hard"
    return rating_string

def update_deckstats(user_id,deck_id,last_reviewed,score):
    deckstat = db.session.query(DeckStat).filter(DeckStat.user_id==user_id).filter(DeckStat.deck_id==deck_id).first()
    if deckstat == None:
        new_deckstat = DeckStat(user_id=user_id,deck_id=deck_id,last_reviewed=str(last_reviewed),average_score=str(score),times_reviewed="1")
        db.session.add(new_deckstat)
        db.session.commit()
    else:
        new_average_score = str(round((float(deckstat.average_score)*int(deckstat.times_reviewed)+float(score))/(int(deckstat.times_reviewed)+1),2))
        new_times_reviewed = str(int(deckstat.times_reviewed)+1)
        db.session.query(DeckStat).filter(DeckStat.user_id==user_id).filter(DeckStat.deck_id==deck_id).update({'last_reviewed':last_reviewed,'average_score':new_average_score,'times_reviewed':new_times_reviewed})
        db.session.commit()

def delete_deckstats(deck_id):
    db.session.query(DeckStat).filter(DeckStat.deck_id==deck_id).delete()
    db.session.commit()

def update_participation(user_id):
    older = db.session.query(Participation).filter(Participation.user_id==user_id).first()
    now = str(int(time.time()))
    if older == None:
        participation = Participation(user_id=user_id,last_revised=now)
        db.session.add(participation)
        db.session.commit()
    else:
        db.session.query(Participation).filter(Participation.user_id==user_id).update({'last_revised':now})
        db.session.commit()

def to_remind(user_id):
    participation = db.session.query(Participation).filter(Participation.user_id==user_id).first()
    if participation == None:
        return true
    else:
        now = int(time.time())
        last_revised = int(participation.last_revised)
        if now-last_revised>=61200:
            return true
        else:
            return false

def get_all_user_ids():
    users = db.session.query(User).all()
    return users


import json
import os
import time

with open("client/backend/test/olimpiadirobotica-2021-default-rtdb-export.json") as db_json:
    db_all = json.load(db_json)

db1 = db_all["db1"]
db1_new = {}
print(str(db1))
for data_type in db1:
    db1_new[data_type] = {"data":{}}
    for date in db1[data_type]["data"]:
        new_date = date.split("-")[2] + "-" + date.split("-")[1] + "-" + date.split("-")[0]
        print(data_type)
        print(new_date)
        db1_new[data_type]["data"][new_date] = db1[data_type]["data"][date]
        #del db1_new[data_type]["data"][date]
print(str(db1_new))

db_all["db1"] = db1_new

with open("client/backend/test/updatedb.json", "w") as update_db:
    json.dump(db_all, update_db)


import functools
import os
import tempfile
from pathlib import Path
from typing import TYPE_CHECKING, Callable, Dict, Optional, Type, Tuple, Union

from ray.air import session
from ray.air.checkpoint import Checkpoint
from ray.air.config import DatasetConfig, RunConfig, ScalingConfig
from ray.train.torch import TorchConfig
from ray.train.trainer import GenDataset

from ray.train.torch import TorchTrainer, get_device
from ray.train.torch.config import _set_torch_distributed_env_vars

ACCELERATE_IMPORT_ERROR: Optional[ImportError] = None

try:
    from ray.train.huggingface.accelerate._accelerate_utils import (
        launch_command,
        AccelerateDefaultNamespace,
        AccelerateConfigWrapper,
        load_accelerate_config,
    )
except ImportError as e:
    ACCELERATE_IMPORT_ERROR = e
    launch_command = None
    AccelerateDefaultNamespace = None
    AccelerateConfigWrapper = None
    load_accelerate_config = None

if TYPE_CHECKING:
    from ray.data.preprocessor import Preprocessor
    from ray.tune.trainable import Trainable

class AccelerateTrainer(TorchTrainer):
    """A Trainer for data parallel HuggingFace Accelerate training with PyTorch.

    This Trainer is a wrapper around the :class:`~ray.train.torch.TorchTrainer`,
    providing the following extra functionality:
    1. Loading and parsing of Accelerate configuration files (created by
    ``accelerate config`` CLI command),
    2. Applying the configuration files on all workers, making sure the environment
    is set up correctly.

    This Trainer runs the function ``train_loop_per_worker`` on multiple Ray
    Actors. These actors already have the necessary torch process group
    configured for distributed PyTorch training, as well as all environment variables
    required by Accelerate, as defined in the configuration file. This allows you
    to use Accelerate APIs (such as ``Accelerator``) inside ``train_loop_per_worker``
    as you would without Ray.

    Inside the ``train_loop_per_worker`` function, In addition to Accelerate APIs, you
    can use any of the :ref:`Ray AIR session methods <air-session-ref>`.
    See full example code below.

    .. testcode::

        def train_loop_per_worker():
            # Report intermediate results for callbacks or logging and
            # checkpoint data.
            session.report(...)

            # Get dict of last saved checkpoint.
            session.get_checkpoint()

            # Session returns the Dataset shard for the given key.
            session.get_dataset_shard("my_dataset")

            # Get the total number of workers executing training.
            session.get_world_size()

            # Get the rank of this worker.
            session.get_world_rank()

            # Get the rank of the worker on the current node.
            session.get_local_rank()

    For more information, see the documentation of
    :class:`~ray.train.torch.TorchTrainer`.

    .. note::

        You need to use ``session.report()`` to communicate results and checkpoints
        back to Ray Train.

    Accelerate integrations with DeepSpeed, FSDP, MegatronLM etc. are fully supported.
    If the Accelerate configuration contains a path to a DeepSpeed config file
    (``deepspeed_config_file``), that file will also be loaded and applied on the
    workers.

    The following Accelerate configuration options will be ignored and automatically
    set by the Trainer according to Ray AIR configs (eg. ``ScalingConfig``):
    - Number of machines (``num_machines``)
    - Number of processes (``num_processes``)
    - Rank of the current machine (``machine_rank``)
    - Local rank of the current machine
    - GPU IDs (``gpu_ids``)
    - Number of CPU threads per process (``num_cpu_threads_per_process``)
    - IP of the head process (``main_process_ip``)
    - Port of the head process (``main_process_port``)
    - Whether all machines are on the same network (``same_network``)
    - Whether to force a CPU-only mode (``cpu``/``use_cpu``)
    - rdzv backend (``rdzv_backend``)
    - Main training function (``main_training_function``)
    - Type of launcher

    This Trainer requires ``accelerate>=0.17.0`` package.
    It is tested with ``accelerate==0.17.1``.

    Example:
        .. testcode::

            import torch
            import torch.nn as nn

            from accelerate import Accelerator

            import ray
            from ray.air import session, Checkpoint
            from ray.train.huggingface import AccelerateTrainer
            from ray.air.config import ScalingConfig
            from ray.air.config import RunConfig
            from ray.air.config import CheckpointConfig

            # If using GPUs, set this to True.
            use_gpu = False

            # Define NN layers archicture, epochs, and number of workers
            input_size = 1
            layer_size = 32
            output_size = 1
            num_epochs = 200
            num_workers = 3

            # Define your network structure
            class NeuralNetwork(nn.Module):
                def __init__(self):
                    super(NeuralNetwork, self).__init__()
                    self.layer1 = nn.Linear(input_size, layer_size)
                    self.relu = nn.ReLU()
                    self.layer2 = nn.Linear(layer_size, output_size)

                def forward(self, input):
                    return self.layer2(self.relu(self.layer1(input)))

            # Define your train worker loop
            def train_loop_per_worker():

                # Initialize the Accelerator
                accelerator = Accelerator()

                # Fetch training set from the session
                dataset_shard = session.get_dataset_shard("train")
                model = NeuralNetwork()

                # Loss function, optimizer, prepare model for training.
                # This moves the data and prepares model for distributed
                # execution
                loss_fn = nn.MSELoss()
                optimizer = torch.optim.Adam(
                    model.parameters(), lr=0.01, weight_decay=0.01
                )
                model, optimizer = accelerator.prepare(model, optimizer)

                # Iterate over epochs and batches
                for epoch in range(num_epochs):
                    for batches in dataset_shard.iter_torch_batches(
                        batch_size=32, dtypes=torch.float
                    ):

                        # Add batch or unsqueeze as an additional dimension [32, x]
                        inputs, labels = torch.unsqueeze(batches["x"], 1), batches["y"]
                        output = model(inputs)

                        # Make output shape same as the as labels
                        loss = loss_fn(output.squeeze(), labels)

                        # Zero out grads, do backward, and update optimizer
                        optimizer.zero_grad()
                        accelerator.backward(loss)
                        optimizer.step()

                        # Print what's happening with loss per 30 epochs
                        if epoch % 20 == 0:
                            print(f"epoch: {epoch}/{num_epochs}, loss: {loss:.3f}")

                    # Report and record metrics, checkpoint model at end of each
                    # epoch
                    session.report(
                        {"loss": loss.item(), "epoch": epoch},
                        checkpoint=Checkpoint.from_dict(
                            dict(
                                epoch=epoch,
                                model=accelerator.unwrap_model(model).state_dict(),
                            )
                        ),
                    )

            torch.manual_seed(42)
            train_dataset = ray.data.from_items(
                [{"x": x, "y": 2 * x + 1} for x in range(200)]
            )

            # Define scaling and run configs
            scaling_config = ScalingConfig(num_workers=3, use_gpu=use_gpu)
            run_config = RunConfig(checkpoint_config=CheckpointConfig(num_to_keep=1))

            trainer = AccelerateTrainer(
                train_loop_per_worker=train_loop_per_worker,
                # Instead of using a dict, you can run ``accelerate config``.
                # The default value of None will then load that configuration
                # file.
                accelerate_config={},
                scaling_config=scaling_config,
                run_config=run_config,
                datasets={"train": train_dataset},
            )

            result = trainer.fit()

            best_checkpoint_loss = result.metrics["loss"]

            # Assert loss is less 0.09
            assert best_checkpoint_loss <= 0.09

    .. testoutput::
        :hide:
        :options: +ELLIPSIS

        ...

    Args:
        train_loop_per_worker: The training function to execute.
            This can either take in no arguments or a ``config`` dict.
        train_loop_config: Configurations to pass into
            ``train_loop_per_worker`` if it accepts an argument.
        accelerate_config: Accelerate configuration to be applied on every worker.
            This can be a path to a file generated with ``accelerate config``,
            a configuration dict or None, in which case it will load the configuration
            file from the default location as defined by Accelerate.
        torch_config: Configuration for setting up the PyTorch backend. If set to
            None, use the default configuration. This replaces the ``backend_config``
            arg of ``DataParallelTrainer``.
        scaling_config: Configuration for how to scale data parallel training.
        dataset_config: Configuration for dataset ingest.
        run_config: Configuration for the execution of the training run.
        datasets: Any Datasets to use for training. Use
            the key "train" to denote which dataset is the training
            dataset. If a ``preprocessor`` is provided and has not already been fit,
            it will be fit on the training dataset. All datasets will be transformed
            by the ``preprocessor`` if one is provided.
        preprocessor: A ``ray.data.Preprocessor`` to preprocess the
            provided datasets.
        resume_from_checkpoint: A checkpoint to resume training from.
    """

    def __init__(
        self,
        train_loop_per_worker: Union[Callable[[], None], Callable[[Dict], None]],
        *,
        train_loop_config: Optional[Dict] = None,
        accelerate_config: Optional[Union[dict, str, Path, os.PathLike]] = None,
        torch_config: Optional[TorchConfig] = None,
        scaling_config: Optional[ScalingConfig] = None,
        dataset_config: Optional[Dict[str, DatasetConfig]] = None,
        run_config: Optional[RunConfig] = None,
        datasets: Optional[Dict[str, GenDataset]] = None,
        preprocessor: Optional["Preprocessor"] = None,
        resume_from_checkpoint: Optional[Checkpoint] = None,
    ):

        if ACCELERATE_IMPORT_ERROR is not None:
            raise ACCELERATE_IMPORT_ERROR

        self.accelerate_config = accelerate_config
        (
            self._accelerate_config_raw,
            self._deepspeed_config_file_raw,
        ) = self._unwrap_accelerate_config_if_needed(accelerate_config)

        super().__init__(
            train_loop_per_worker=train_loop_per_worker,
            train_loop_config=train_loop_config,
            torch_config=torch_config,
            scaling_config=scaling_config,
            dataset_config=dataset_config,
            run_config=run_config,
            datasets=datasets,
            preprocessor=preprocessor,
            resume_from_checkpoint=resume_from_checkpoint,
        )

    def _unwrap_accelerate_config_if_needed(
        self,
        accelerate_config: Optional[
            Union[dict, str, Path, os.PathLike, AccelerateConfigWrapper]
        ],
    ) -> Tuple[str, Optional[str]]:
        # The AccelerateConfigWrapper is used to deal with the issue of the
        # Trainer being initialized twice (by the user and by us in the Trainable).
        # If it's initialized by the user, accelerate_config will not be an instance
        # of AccelerateConfigWrapper. This means we should read the config file from
        # given path.
        # If accelerate_config is an instance of AccelerateConfigWrapper, that means
        # we are dealing with a file that was already read, and we should instead use
        # the string in the wrapper as the raw contents of the file. This should
        # only happen internally, during initialization of this class in the Trainable.
        if isinstance(accelerate_config, AccelerateConfigWrapper):
            return (
                accelerate_config.config_raw,
                accelerate_config.deepspeed_config_raw,
            )
        else:
            return load_accelerate_config(accelerate_config)

    def as_trainable(self) -> Type["Trainable"]:
        # We want to load the config when the Trainer is first instantiated,
        # and share the contents with the Trainables (which may be on different)
        # nodes
        old_accelerate_config = self._param_dict.get("accelerate_config", None)
        self._param_dict["accelerate_config"] = AccelerateConfigWrapper(
            self._accelerate_config_raw, self._deepspeed_config_file_raw
        )
        try:
            ret = super().as_trainable()
        finally:
            self._param_dict["accelerate_config"] = old_accelerate_config
        return ret

    def training_loop(self) -> None:
        old_train_loop_per_worker = self._train_loop_per_worker
        self._train_loop_per_worker = self._wrap_train_loop_per_worker(
            self._train_loop_per_worker,
            self._accelerate_config_raw,
            self._deepspeed_config_file_raw,
        )
        try:
            ret = super().training_loop()
        finally:
            self._train_loop_per_worker = old_train_loop_per_worker
        return ret

    @classmethod
    def _wrap_train_loop_per_worker(
        cls,
        train_loop_per_worker: Union[Callable[[], None], Callable[[Dict], None]],
        accelerate_config_raw: str,
        deepspeed_config_file_raw: str,
    ):
        """Wrap around train_loop_per_worker to set necessary Accelerate env vars."""

        @functools.wraps(train_loop_per_worker)
        def _accelerate_train_loop_per_worker(*args, **kwargs):
            with tempfile.TemporaryDirectory() as tempdir:
                # Write Accelerate config to file so it can be read
                # by Accelerate
                temp_config_file = os.path.join(tempdir, "default_config.yaml")
                with open(temp_config_file, "w") as f:
                    f.write(accelerate_config_raw)

                # Set by TorchBackend
                master_addr = os.environ["MASTER_ADDR"]
                master_port = os.environ["MASTER_PORT"]

                namespace = AccelerateDefaultNamespace()
                namespace.config_file = temp_config_file
                namespace.num_processes = 1
                namespace.num_machines = session.get_world_size()
                namespace.machine_rank = session.get_world_rank()
                namespace.num_cpu_threads_per_process = (
                    session.get_trial_resources().bundles[-1].get("CPU", 1)
                )
                namespace.gpu_ids = None
                namespace.main_process_ip = master_addr
                namespace.main_process_port = master_port
                namespace.same_network = False

                device = get_device()
                if isinstance(device, list):
                    device = device[0]
                if device.type == "cpu":
                    os.environ["LOCAL_RANK"] = "-1"
                    namespace.use_cpu = True
                else:
                    namespace.use_cpu = False

                # Handle DeepSpeed config
                if isinstance(deepspeed_config_file_raw, dict):
                    namespace.deepspeed_config_file = deepspeed_config_file_raw
                elif deepspeed_config_file_raw:
                    deepspeed_config_file = os.path.join(
                        tempdir, "deepspeed_config.json"
                    )
                    with open(deepspeed_config_file, "w") as f:
                        f.write(deepspeed_config_file_raw)
                    namespace.deepspeed_config_file = deepspeed_config_file

                # Let Accelerate set all env vars
                launch_command(namespace)

                # Set our env vars again to override Accelerate
                os.environ["MASTER_ADDR"] = master_addr
                os.environ["MASTER_PORT"] = master_port
                _set_torch_distributed_env_vars()

                if device.type == "cpu":
                    os.environ["LOCAL_RANK"] = "-1"

                return train_loop_per_worker(*args, **kwargs)

        return _accelerate_train_loop_per_worker


#!/usr/bin/env python3

import asyncio
import sys

#该协程函数使用了递归。
async def stopwatch(sec):
    await asyncio.sleep(1)
    if sec <= 0:
        sys.stdout.write('0\n')
        sys.stdout.flush()
        return
    else:
        sys.stdout.write(str(sec) + ' ')
        sys.stdout.flush()
        await(stopwatch(sec -1))

async def main():
    if len(sys.argv) < 2:
        sys.exit(1)
    try:
        sec = int(sys.argv[1])
        await stopwatch(sec)
        sys.exit(0)
    except Exception:
        sys.exit(1)

if __name__ == '__main__':
    asyncio.run(main())