from __future__ import print_function

import argparse
import csv
import os

import numpy as np
import torch
from torch.autograd import Variable
import torch.backends.cudnn as cudnn
import torch.nn as nn
import torch.optim as optim
import torchvision.transforms as transforms
import torchvision.datasets as datasets

import torchvision.models as models
from utils import progress_bar
from torchvision.utils import save_image
from tensorboardX import SummaryWriter
import matplotlib.pyplot as plt
from sklearn.metrics import confusion_matrix

parser = argparse.ArgumentParser(description='PyTorch CIFAR10 Training')

parser.add_argument("--train_dir", required=True, type=str, help="Path to train data parent directory.")
parser.add_argument("--test_dir", required=True, type=str, help="Path to test data parent directory.")
parser.add_argument('--lr', default=0.1, type=float, help='learning rate')
parser.add_argument('--resume', '-r', action='store_true',
                    help='resume from checkpoint')
parser.add_argument('--model', default="ResNet18", type=str,
                    help='model type (default: ResNet50)')
parser.add_argument('--name', default='0', type=str, help='name of run')
parser.add_argument('--seed', default=0, type=int, help='random seed')
parser.add_argument('--batch_size', default=128, type=int, help='batch size')
parser.add_argument('--epoch', default=200, type=int,
                    help='total epochs to run')
parser.add_argument('--no-augment', dest='augment', action='store_false',
                    help='use standard augmentation (default: True)')
parser.add_argument('--decay', default=1e-4, type=float, help='weight decay')
parser.add_argument('--alpha', default=1., type=float,
                    help='mixup interpolation coefficient (default: 1)')
args = parser.parse_args()

use_cuda = torch.cuda.is_available()

writer = SummaryWriter()

best_acc = 0  # best test accuracy
start_epoch = 0  # start from epoch 0 or last checkpoint epoch
save_dir = "results"

if args.seed != 0:
    torch.manual_seed(args.seed)

# Data
print('==> Preparing data..')
if args.augment:
    transform_train = transforms.Compose([
        transforms.Resize((128,128)),
        #transforms.RandomCrop(32, padding=4),
        #transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        transforms.Normalize((0.4914, 0.4822, 0.4465),
                             (0.2023, 0.1994, 0.2010)),
    ])
else:
    transform_train = transforms.Compose([
        transforms.Resize((128,128)),
        transforms.ToTensor(),
        transforms.Normalize((0.4914, 0.4822, 0.4465),
                             (0.2023, 0.1994, 0.2010)),
    ])

transform_test = transforms.Compose([
    transforms.Resize((128,128)),
    transforms.ToTensor(),
    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])

trainset = datasets.ImageFolder(os.path.abspath(args.train_dir), transform=transform_train)
trainloader = torch.utils.data.DataLoader(trainset,
                                          batch_size=args.batch_size,
                                          shuffle=True, num_workers=3)

testset = datasets.ImageFolder(os.path.abspath(args.test_dir), transform=transform_test)
testloader = torch.utils.data.DataLoader(testset, batch_size=args.batch_size,
                                         shuffle=False, num_workers=3)

print('==> Building model..')
device = 'cuda' if torch.cuda.is_available() else 'cpu'
#net = models.resnet50(pretrained=True)
net = models.resnet50(weights='ResNet50_Weights.DEFAULT')
modules = list(net.children())[:-1]
modules.append(nn.Flatten())
modules.append(nn.Linear(net.fc.in_features,4))
net = nn.Sequential(*modules)
net = net.to(device)

if device == 'cuda':
    net = torch.nn.DataParallel(net)
    cudnn.benchmark = True

"""

# Model
if args.resume:
    # Load checkpoint.
    print('==> Resuming from checkpoint..')
    assert os.path.isdir('checkpoint'), 'Error: no checkpoint directory found!'
    checkpoint = torch.load('./checkpoint/ckpt.t7' + args.name + '_'
                            + str(args.seed))
    net = checkpoint['net']
    best_acc = checkpoint['acc']
    start_epoch = checkpoint['epoch'] + 1
    rng_state = checkpoint['rng_state']
    torch.set_rng_state(rng_state)
else:
    print('==> Building model..')
    net = models.__dict__[args.model]()

if not os.path.isdir('results'):
    os.mkdir('results')
logname = ('results/log_' + net.__class__.__name__ + '_' + args.name + '_'
           + str(args.seed) + '.csv')

if use_cuda:
    net.cuda()
    net = torch.nn.DataParallel(net)
    print(torch.cuda.device_count())
    cudnn.benchmark = True
    print('Using CUDA..')
"""

criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(), lr=args.lr, momentum=0.9, weight_decay=args.decay)

def save_model(model):
    torch.save(model.state_dict(), save_dir + "/"+str(args.seed)+"model.pt")

def CreateDir(path):
        try:
                os.mkdir(path)
        except OSError as error:
                print(error)

CreateDir(save_dir)

def mixup_data(x, y, alpha=1.0, use_cuda=True):
    '''Returns mixed inputs, pairs of targets, and lambda'''
    if alpha > 0:
        lam = np.random.beta(alpha, alpha)
    else:
        lam = 1

    batch_size = x.size()[0]
    if use_cuda:
        index = torch.randperm(batch_size).cuda()
    else:
        index = torch.randperm(batch_size)

    mixed_x = lam * x + (1 - lam) * x[index, :]
    y_a, y_b = y, y[index]
    return mixed_x, y_a, y_b, lam

def mixup_criterion(criterion, pred, y_a, y_b, lam):
    return lam * criterion(pred, y_a) + (1 - lam) * criterion(pred, y_b)

def train(epoch):
    print('\nEpoch: %d' % epoch)
    net.train()
    train_loss = 0
    reg_loss = 0
    correct = 0
    total = 0
    for batch_idx, (inputs, targets) in enumerate(trainloader):
        if use_cuda:
            inputs, targets = inputs.cuda(), targets.cuda()

        inputs_mixup, targets_a, targets_b, lam = mixup_data(inputs, targets,
                                                       args.alpha, use_cuda)
        inputs_mixup, targets_a, targets_b = map(Variable, (inputs_mixup,
                                                      targets_a, targets_b))

        outputs = net(inputs_mixup)

        loss = mixup_criterion(criterion, outputs, targets_a, targets_b, lam)
        train_loss += loss
        _, predicted = torch.max(outputs.data, 1)
        total += targets.size(0)
        correct += (lam * predicted.eq(targets_a.data).sum().float()
                    + (1 - lam) * predicted.eq(targets_b.data).sum().float())

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        progress_bar(batch_idx, len(trainloader),
                     'Loss: %.3f | Reg: %.5f | Acc: %.3f%% (%d/%d)'
                     % (train_loss/(batch_idx+1), reg_loss/(batch_idx+1),
                        100.*correct/total, correct, total))
        writer.add_scalar('Training/ACC',100.*correct/total, (epoch*len(trainloader.dataset)/args.batch_size)+batch_idx)
        writer.add_scalar('Training/loss',train_loss/(batch_idx+1),(epoch*len(trainloader.dataset)/args.batch_size)+batch_idx)
    return (train_loss/batch_idx, reg_loss/batch_idx, 100.*correct/total)

def test(epoch):
    global best_acc
    net.eval()
    test_loss = 0
    correct = 0
    total = 0
    for batch_idx, (inputs, targets) in enumerate(testloader):
        if use_cuda:
            inputs, targets = inputs.cuda(), targets.cuda()
        inputs, targets = Variable(inputs, volatile=False), Variable(targets)
        outputs = net(inputs)

        _, predicted = torch.max(outputs.data, 1)
        total += targets.size(0)
        correct += predicted.eq(targets.data).cpu().sum()

        progress_bar(batch_idx, len(testloader),
                     'Loss: %.3f | Acc: %.3f%% (%d/%d)'
                     % (test_loss/(batch_idx+1), 100.*correct/total,
                        correct, total))
    acc = 100.*correct/total
    writer.add_scalar('Test/ACC',acc, (epoch*len(testloader.dataset)/args.batch_size)+batch_idx)
    if epoch == start_epoch + args.epoch - 1 or acc > best_acc:
        checkpoint(acc, epoch)
    if acc > best_acc:
        best_acc = acc
    return (test_loss/batch_idx, 100.*correct/total)

def val(epoch):
    global best_acc
    net.eval()
    test_loss = 0
    correct = 0
    total = 0
    pred_ = []
    label_ = []
    for batch_idx, (inputs, targets) in enumerate(testloader):
        if use_cuda:
            inputs, targets = inputs.cuda(), targets.cuda()
        inputs, targets = Variable(inputs, volatile=False), Variable(targets)
        outputs = net(inputs)

        _, predicted = torch.max(outputs.data, 1)

        pred_.append(predicted.data.cpu().numpy())
        label_.append(targets.data.cpu().numpy())

        total += targets.size(0)
        correct += predicted.eq(targets.data).cpu().sum()

        progress_bar(batch_idx, len(testloader),
                     'Loss: %.3f | Acc: %.3f%% (%d/%d)'
                     % (test_loss/(batch_idx+1), 100.*correct/total,
                        correct, total))
    acc = 100.*correct/total
    if epoch == start_epoch + args.epoch - 1 or acc > best_acc:
        checkpoint(acc, epoch)
    if acc > best_acc:
        best_acc = acc
    return pred_, label_, (test_loss/batch_idx), (100.*correct/total)

def convert_label_(pred_y,true_l,list_label):
  y_true = []
  y_pred = []
  for i in range(len(pred_y)):
    for j in range(len(pred_y[i])):
      y_pred.append(list_label[pred_y[i][j].item()])
      y_true.append(list_label[true_l[i][j].item()])
  return np.array(y_true),np.array(y_pred)

def plot_confusion_matrix(cm,
                          target_names,
                          title='Confusion matrix',
                          titleG='Confusion matrix',
                          cmap=None,
                          normalize=True):
    """
    given a sklearn confusion matrix (cm), make a nice plot

    Arguments
    ---------
    cm:           confusion matrix from sklearn.metrics.confusion_matrix

    target_names: given classification classes such as [0, 1, 2]
                  the class names, for example: ['high', 'medium', 'low']

    title:        the text to display at the top of the matrix

    cmap:         the gradient of the values displayed from matplotlib.pyplot.cm
                  see http://matplotlib.org/examples/color/colormaps_reference.html
                  plt.get_cmap('jet') or plt.cm.Blues

    normalize:    If False, plot the raw numbers
                  If True, plot the proportions

    Usage
    -----
    plot_confusion_matrix(cm           = cm,                  # confusion matrix created by
                                                              # sklearn.metrics.confusion_matrix
                          normalize    = True,                # show proportions
                          target_names = y_labels_vals,       # list of names of the classes
                          title        = best_estimator_name) # title of graph

    Citiation
    ---------
    http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html

    """
    import matplotlib.pyplot as plt_
    import numpy as np
    import itertools

    accuracy = np.trace(cm) / float(np.sum(cm))
    misclass = 1 - accuracy

    if cmap is None:
        cmap = plt_.get_cmap('Blues')

    plt_.figure()
    plt_.imshow(cm, interpolation='nearest', cmap=cmap)
    plt_.title(titleG)
    plt_.colorbar()

    if target_names is not None:
        tick_marks = np.arange(len(target_names))
        plt_.xticks(tick_marks, target_names, rotation=45)
        plt_.yticks(tick_marks, target_names)

    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]

    thresh = cm.max() / 1.5 if normalize else cm.max() / 2
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        if normalize:
            plt_.text(j, i, "{:0.4f}".format(cm[i, j]),
                     horizontalalignment="center",
                     color="white" if cm[i, j] > thresh else "black")
        else:
            plt_.text(j, i, "{:,}".format(cm[i, j]),
                     horizontalalignment="center",
                     color="white" if cm[i, j] > thresh else "black")

    plt_.tight_layout()
    plt_.ylabel('Verdade do label')
    plt_.xlabel('Predicted label')
    plt_.xlabel('Predição do  label\naccuracy={:0.4f}; misclass={:0.4f}'.format(accuracy, misclass))
    plt_.savefig(title+'-confusion_matrix.png', dpi=120)
    plt_.close()
    #plt.show()

def checkpoint(acc, epoch):
    # Save checkpoint.
    print('Saving..')
    state = {
        'net': net,
        'acc': acc,
        'epoch': epoch,
        'rng_state': torch.get_rng_state()
    }
    if not os.path.isdir('checkpoint'):
        os.mkdir('checkpoint')
    torch.save(state, './checkpoint/ckpt.t7' + args.name + '_'
               + str(args.seed))

def adjust_learning_rate(optimizer, epoch):
    """decrease the learning rate at 100 and 150 epoch"""
    lr = args.lr
    if epoch >= 100:
        lr /= 10
    if epoch >= 150:
        lr /= 10
    for param_group in optimizer.param_groups:
        param_group['lr'] = lr

"""
if not os.path.exists(logname):
    with open(logname, 'w') as logfile:
        logwriter = csv.writer(logfile, delimiter=',')
        logwriter.writerow(['epoch', 'train loss', 'reg loss', 'train acc',
                            'test loss', 'test acc'])
"""
for epoch in range(start_epoch, args.epoch):
    train_loss, reg_loss, train_acc = train(epoch)
    test_loss, test_acc = test(epoch)
    adjust_learning_rate(optimizer, epoch)
    #with open(logname, 'a') as logfile:
        #logwriter = csv.writer(logfile, delimiter=',')
        #logwriter.writerow([epoch, train_loss, reg_loss, train_acc, test_loss,
                            #test_acc])

save_model(net)

pred, label, test_loss, test_acc = val(epoch)

print(test_acc)
print(testset.classes)

y_pred, y_true = convert_label_(pred,label,testset.classes)
confusion = confusion_matrix(y_true, y_pred, labels = testset.classes)

print(confusion)
plot_confusion_matrix(cm = np.array(confusion),normalize = False, target_names = testset.classes, title =' Test', titleG = ' Test')


"""
    Computations for GISANS
"""

import logging
from multiprocessing import Pool

import numpy as np

H_OVER_M_NEUTRON = 3.956034e-7  # h/m_n [m^2/s]

class GISANS(object):
    """
    Compute grazing-incident SANS
    """

    def __init__(self, cross_section_data):
        """
        :param CrossSectionData cross_section_data: processed data object

        The calculations here are meant to match QuickNXS v1. The following are
        items to improve on:
            - Background subtraction
            - Trim the TOF distribution to the chopper bandwidth

        """
        self.data_set = cross_section_data

    def __call__(self, direct_beam=None):
        x_pos = self.data_set.configuration.peak_position
        y_pos = self.data_set.configuration.low_res_position
        scale = 1.0 / self.data_set.proton_charge * self.data_set.configuration.scaling_factor

        rad_per_pixel = self.data_set.det_size_x / self.data_set.dist_sam_det / self.data_set.xydata.shape[1]

        # QuickNXS v1 uses the whole detector for GISANS calculations and doesn't trim the edges
        # active_area_x = self.data_set.active_area_x
        # active_area_y = self.data_set.active_area_y
        active_area_x = [0, 304]
        active_area_y = [0, 256]
        xtth = self.data_set.direct_pixel - np.arange(self.data_set.data.shape[0])[active_area_x[0] : active_area_x[1]]
        pix_offset_spec = self.data_set.direct_pixel - x_pos
        delta_dangle = self.data_set.dangle - self.data_set.angle_offset
        tth_spec = delta_dangle * np.pi / 180.0 + pix_offset_spec * rad_per_pixel
        af = delta_dangle * np.pi / 180.0 + xtth * rad_per_pixel - tth_spec / 2.0
        ai = np.ones_like(af) * tth_spec / 2.0

        phi = (np.arange(self.data_set.data.shape[1])[active_area_y[0] : active_area_y[1]] - y_pos) * rad_per_pixel

        # To be compatible with QuickNXS v1, take the whole TOF range rather than trimming the edges
        # ws = self.data_set.event_workspace
        # tof_edges = np.arange(ws.getTofMin(), ws.getTofMax(), self.data_set.configuration.tof_bins)
        tof_edges = self.data_set.tof_edges

        v_edges = self.data_set.dist_mod_det / tof_edges * 1e6  # m/s
        lambda_edges = H_OVER_M_NEUTRON / v_edges * 1e10  # A

        wavelengths = np.asarray((lambda_edges[:-1] + lambda_edges[1:]) / 2.0)
        k = 2.0 * np.pi / wavelengths
        # calculate ROI intensities and normalize by number of points
        # Note: the number of points here may need to be adjusted from the specular reflectivity
        # calculation if we used a final rebining.
        P0 = self.data_set.configuration.cut_first_n_points
        PN = len(self.data_set.tof) - self.data_set.configuration.cut_last_n_points
        self.wavelengths = wavelengths[P0:PN]

        # calculate reciprocal space, incident and outgoing perpendicular wave vectors
        self.Qy = k[np.newaxis, np.newaxis, P0:PN] * (np.sin(phi) * np.cos(af)[:, np.newaxis])[:, :, np.newaxis]
        p_i = k[np.newaxis, np.newaxis, P0:PN] * ((0.0 * phi) + np.sin(ai)[:, np.newaxis])[:, :, np.newaxis]
        self.p_f = k[np.newaxis, np.newaxis, P0:PN] * ((0.0 * phi) + np.sin(af)[:, np.newaxis])[:, :, np.newaxis]
        self.Qz = p_i + self.p_f

        raw = self.data_set.data[active_area_x[0] : active_area_x[1], active_area_y[0] : active_area_y[1], P0:PN]

        intensity = scale * np.array(raw)
        d_intensity = scale * np.sqrt(raw)

        if direct_beam is not None:
            if not direct_beam.configuration.tof_bins == self.data_set.configuration.tof_bins:
                logging.error("Trying to normalize with a direct beam data set with different binning")

            norm_y_min, norm_y_max = direct_beam.configuration.low_res_roi
            norm_x_min, norm_x_max = direct_beam.configuration.peak_roi
            norm_raw_multi_dim = direct_beam.data[norm_x_min:norm_x_max, norm_y_min:norm_y_max, P0:PN]

            norm_raw = norm_raw_multi_dim.sum(axis=0).sum(axis=0)
            norm_d_raw = np.sqrt(norm_raw)
            norm_scale = (float(norm_x_max) - float(norm_x_min)) * (float(norm_y_max) - float(norm_y_min))
            norm_raw /= norm_scale * direct_beam.proton_charge
            norm_d_raw /= norm_scale * direct_beam.proton_charge

            idxs = norm_raw > 0.0
            d_intensity[:, :, idxs] = np.sqrt(
                (d_intensity[:, :, idxs] / norm_raw[idxs][np.newaxis, np.newaxis, :]) ** 2
                + (
                    intensity[:, :, idxs]
                    / norm_raw[idxs][np.newaxis, np.newaxis, :] ** 2
                    * norm_d_raw[idxs][np.newaxis, np.newaxis, :]
                )
                ** 2
            )
            intensity[:, :, idxs] /= norm_raw[idxs][np.newaxis, np.newaxis, :]
            intensity[:, :, np.logical_not(idxs)] = 0.0
            d_intensity[:, :, np.logical_not(idxs)] = 0.0

        self.S = intensity
        self.dS = d_intensity

        # Create plotting data
        # TODO: use options to plot the right data (for instance: qz or pf)
        self.SGrid, qy, qz = np.histogram2d(
            self.Qy.flatten(), self.Qz.flatten(), bins=(50, 50), weights=intensity.flatten()
        )
        npoints, _, _ = np.histogram2d(self.Qy.flatten(), self.Qz.flatten(), bins=(50, 50))
        self.SGrid[npoints > 0] /= npoints[npoints > 0]
        self.SGrid = self.SGrid.transpose()
        qy = (qy[:-1] + qy[1:]) / 2.0
        qz = (qz[:-1] + qz[1:]) / 2.0
        self.QyGrid, self.QzGrid = np.meshgrid(qy, qz)

def merge(reduction_list, pol_state, wl_min=0, wl_max=100):
    """
    Merge the off-specular data from a reduction list.
    :param list reduction_list: list of NexusData objects
    :param string pol_state: polarization state to consider

    The scaling factors should have been determined at this point. Just use them
    to merge the different runs in a set.

    TODO: This doesn't deal with the overlap properly. It assumes that the user
    cut the overlapping points by hand.
    """
    _qy = np.zeros(0)
    _qz = np.zeros(0)
    _pf = np.zeros(0)
    _s = np.zeros(0)
    _ds = np.zeros(0)
    _wl = np.zeros(0)

    for item in reduction_list:
        gisans = item.cross_sections[pol_state].gisans_data
        Qy, Qz, pf, S, dS = (gisans.Qy, gisans.Qz, gisans.p_f, gisans.S, gisans.dS)

        # Filter according to wavelength
        filtered = np.where((gisans.wavelengths >= wl_min) & (gisans.wavelengths <= wl_max))
        _qy = np.concatenate((_qy, Qy[:, :, filtered].flatten()))
        _qz = np.concatenate((_qz, Qz[:, :, filtered].flatten()))
        _pf = np.concatenate((_pf, pf[:, :, filtered].flatten()))
        _s = np.concatenate((_s, S[:, :, filtered].flatten()))
        _ds = np.concatenate((_ds, dS[:, :, filtered].flatten()))
        _wl = np.concatenate((_wl, gisans.wavelengths[filtered]))

    return _qy, _qz, _pf, _s, _ds, _wl

def rebin_extract(reduction_list, pol_state, wl_min, wl_max, qy_npts=50, qz_npts=50, use_pf=False):

    binning = (qy_npts + 1, qz_npts + 1)
    qy, qz, pf, intensity, d_intensity, _ = merge(reduction_list, pol_state, wl_min=wl_min, wl_max=wl_max)
    if use_pf:
        _z_axis = pf
    else:
        _z_axis = qz

    n_points, _qy, _qz_axis = np.histogram2d(qy, _z_axis, bins=binning)
    _intensity_summed, _, _ = np.histogram2d(qy, _z_axis, bins=(_qy, _qz_axis), weights=intensity)
    _intensity_err, _, _ = np.histogram2d(qy, _z_axis, bins=(_qy, _qz_axis), weights=d_intensity**2)

    _intensity_summed[n_points > 0] /= n_points[n_points > 0]
    _intensity_err = np.sqrt(_intensity_err)
    _intensity_err[n_points > 0] /= n_points[n_points > 0]

    _qy = (_qy[:-1] + _qy[1:]) / 2.0
    _qz_axis = (_qz_axis[:-1] + _qz_axis[1:]) / 2.0

    return _intensity_summed, _qy, _qz_axis, _intensity_err

def _rebin_proc(data):
    # Filter data
    wl = data["wl"]
    filtered = np.where((wl >= data["wl_min"]) & (wl <= data["wl_max"]))
    qy = data["qy"][filtered]
    _z_axis = data["qz"][filtered]
    intensity = data["intensity"][filtered]
    d_intensity = data["d_intensity"][filtered]

    # Perform binning
    n_points, _qy, _qz_axis = np.histogram2d(qy, _z_axis, bins=data["binning"])
    _intensity_summed, _, _ = np.histogram2d(qy, _z_axis, bins=(_qy, _qz_axis), weights=intensity)
    _intensity_err, _, _ = np.histogram2d(qy, _z_axis, bins=(_qy, _qz_axis), weights=d_intensity**2)

    _intensity_summed[n_points > 0] /= n_points[n_points > 0]
    _intensity_err = np.sqrt(_intensity_err)
    _intensity_err[n_points > 0] /= n_points[n_points > 0]

    _qy = (_qy[:-1] + _qy[1:]) / 2.0
    _qz_axis = (_qz_axis[:-1] + _qz_axis[1:]) / 2.0

    return _intensity_summed, _qy, _qz_axis, _intensity_err

def rebin_parallel(reduction_list, pol_state, wl_min, wl_max, wl_npts=2, qy_npts=50, qz_npts=50, use_pf=False):
    """
    Process the wavelength bands in parallel.
    """
    # First, merge all the data
    binning = (qy_npts + 1, qz_npts + 1)
    qy, qz, pf, intensity, d_intensity, wl_array = merge(reduction_list, pol_state, wl_min=0, wl_max=100.0)
    if use_pf:
        _z_axis = pf
    else:
        _z_axis = qz

    # Create one job per wavelength band
    inputs = []
    for i in range(wl_npts):
        wl_step = (wl_max - wl_min) / wl_npts
        _wl_min = wl_min + i * wl_step
        _wl_max = wl_min + (i + 1) * wl_step
        _d = dict(
            qy=qy,
            qz=_z_axis,
            intensity=intensity,
            d_intensity=d_intensity,
            wl=wl_array,
            wl_min=_wl_min,
            wl_max=_wl_max,
            binning=binning,
        )
        inputs.append(_d)

    pool = Pool(wl_npts)
    results = pool.map(_rebin_proc, inputs)
    return results


#
# Task 4: estimate alpha & beta values.
#

import argparse
import math
import numpy as np
from sklearn.linear_model import LinearRegression

def get_tensor_size(size_str):
    if size_str.endswith("MB"):
        return float(size_str[:-2])
    if size_str.endswith("KB"):
        return float(size_str[:-2]) / 1024
    if size_str.endswith("B"):
        return float(size_str[:-1]) / (1024 * 1024)
    return float(size_str)

def read_task2(csv):
    result = {
        'vsizes': [],
        'ring': [],
        'recur_hd': []
    }

    with open(csv, 'r') as fcsv:
        curr_run = []

        for line in fcsv.readlines():
            line = line.strip()
            segs = line.split(', ')
            assert len(segs) == 6
            assert segs[0] == "task2-tag"

            size = get_tensor_size(segs[1])
            alg = segs[3]
            time = float(segs[5])

            if len(result['vsizes']) == 0 or size != result['vsizes'][-1]:
                result['vsizes'].append(size)
            curr_run.append(time)

            num_nodes = int(segs[2])
            node_id = int(segs[4])
            if node_id == num_nodes - 1:
                avg = sum(curr_run) / len(curr_run)
                result[alg].append(avg)
                curr_run = []

    return result

def solve_linear_regression(xs, ys):
    xs, ys = np.array(xs), np.array(ys)
    xs = xs.reshape((-1, 1))    # requires 2D X column vector
    model = LinearRegression().fit(xs, ys)
    assert len(model.coef_) == 1

    slope = model.coef_[0]
    intercept = model.intercept_
    r_squared = model.score(xs, ys)
    return slope, intercept, r_squared

def calculate_params_for_alg(alg, num_nodes, slope, intercept):
    if alg == 'ring':
        alpha = intercept / (num_nodes - 1)
        beta = slope / ((num_nodes - 1) / num_nodes)
        return alpha, beta
    else:
        alpha = intercept / math.log(num_nodes, 2)
        beta = slope / math.log(num_nodes, 2)
        return alpha, beta

def estimate_params(result):
    """
    Use linear regression to get estimated values for alpha & beta.

      ring:     T = (p-1) * alpha + ((p-1)/p) * n * beta
                  = ((p-1)/p) * beta * n + (p-1) * alpha
                    ------ a ------- * x + ----- b -----

      recur_hd: T = log(p) * (alpha + n * beta)
                  = log(p) * beta * n + log(p) * alpha
                    ----- a ----- * x + ----- b ------

    number of nodes p = 16 in task2, vector size n measured in MB

    We assume that doing the reduction operation (vector add) is cheap
    enough to be negligible.
    """
    print("alg, alpha, beta_in_MB, r_squared")
    for alg in ('ring', 'recur_hd'):
        xs = result['vsizes']
        ys = result[alg]

        slope, intercept, r_squared = solve_linear_regression(xs, ys)
        alpha, beta = calculate_params_for_alg(alg, 16, slope, intercept)
        print(f"{alg}, {alpha:.6f}, {beta:.6f}, {r_squared:.6f}")

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--task2_csv", required=True, type=str)
    args = parser.parse_args()

    result = read_task2(args.task2_csv)
    estimate_params(result)


import plotly.express as px
import plotly.graph_objects as go
import pandas as pd
import numpy as np
import sys

def get_scatter(df=None):

	if df is None:
		# x = [0, 50, 100, 200, 300, 400, 500, 600]
		# y = [0, 1.5, 2, 4, 7.5, 12.5, 20, 40.6]
		# y1 = [0, 1.7, 3, 5, 8.5, 15.5, 24, 42.6]

		d1 = np.random.normal(5, 0.5, 200)
		d2 = np.random.normal(30, 1.2, 60)
		d3 = np.random.normal(6, 1, 60)
		d11 = np.random.normal(5, 0.5, 200)
		d22 = np.random.normal(30, 1.2, 60)
		d33 = np.random.normal(6, 1, 60)

		y = np.concatenate((d1, d2, d3)).flatten()
		y1 = np.concatenate((d11, d22, d33)).flatten()
		x = np.arange(len(y))

	else:
		x = np.arange(len(df))
		y = df['S1']
		y1 = df['S2']

	fig = go.Figure()
	fig.add_trace(go.Scatter(x=x,
	                         y=y,
	                         name='Sensor 1',
	                         marker_color='#E48F72'))
	fig.add_trace(go.Scatter(x=x,
	                         y=y1,
	                         name='Sensor 2',
	                         marker_color='lightgreen'))

	fig.update_layout(title=dict(
					    x=0.5,
					    y=0.8,
	                    font=dict(size=20, color='black')),
					legend=dict(
				        x=0,
				        y=1,
				        bgcolor = '#373a40',
				        traceorder='normal',
				        font=dict(
				            size=12,
				            color= 'white'),
				    ),
					template='plotly_dark',
					height=330,
					width=800,
					font=dict(family="Courier",
					        size=12, color='black'),
					paper_bgcolor='rgba(0,0,0,0)',
					plot_bgcolor='gray',
				    margin=dict(t=50, b=70, l=80, r=1))
	fig.update_xaxes(title='Time Interval [100ms]')
	fig.update_yaxes(title='PRESSURE in PSI')
	return fig

def get_bar(color='steelblue', title='', df=None):

	SIZE = 10

	if df is None:
	    d1 = np.random.normal(5, 0.5, 200)
	    d2 = np.random.normal(30, 1.2, 60)
	    d3 = np.random.normal(6, 1, 60)

	    data = np.concatenate((d1, d2, d3))
	    df = pd.DataFrame()
	    df['data'] = data
	else:

		data = df.values
		modulo = len(data) % SIZE
		df = pd.DataFrame()
		df['data'] = data
		if modulo> 0:
			df = df[:-modulo]
		df.dropna(inplace = True)

	x = []
	for i in range(int(len(df.data)/SIZE)):
	    for r in range(SIZE):
	    	x.append(i)
	df['bin'] = np.array(x)
	df = df.groupby('bin').max()

	fig = go.Figure()
	fig.add_trace(go.Bar(x=df.index,
	                     y=df.values.flatten(),
	                     name='Sensor 1',
	                     marker_color= color))

	fig.update_layout(title=dict(
						text = "<b>{}</b>".format(title),
				        x=0.6,
				        y= 0.8,
                      	font=dict(size=20, color='black')),
					template='plotly_dark',
					barmode = 'stack',
					height=330,
					width= 400,
					font=dict(family="Courier",
					        size=12, color='black'),
					paper_bgcolor='rgba(0,0,0,0)',
					plot_bgcolor='gray',
				    margin=dict(r=1))
	fig.update_xaxes(title='Time Interval [1s]')
	fig.update_yaxes(title='PRESSURE in PSI')
	return fig


import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F

import config
from attacks.attacks import AttackInstance

def dct(x, norm="ortho"):
    """
    Discrete Cosine Transform, Type II (a.k.a. the DCT)
    For the meaning of the parameter `norm`, see:
    https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.fftpack.dct.html
    :param x: the input signal
    :param norm: the normalization, None or 'ortho'
    :return: the DCT-II of the signal over the last dimension
    """
    x_shape = x.shape
    N = x_shape[-1]
    x = x.reshape(-1, N)

    v = torch.cat([x[:, ::2], x[:, 1::2].flip([1])], dim=1)

    Vc = torch.view_as_real(torch.fft.fft(v, dim=1))

    k = -torch.arange(N, dtype=x.dtype, device=x.device)[None, :] * np.pi / (2 * N)
    W_r = torch.cos(k)
    W_i = torch.sin(k)

    V = Vc[:, :, 0] * W_r - Vc[:, :, 1] * W_i

    if norm == "ortho":
        V[:, 0] /= np.sqrt(N) * 2
        V[:, 1:] /= np.sqrt(N / 2) * 2

    V = 2 * V.view(*x_shape)

    return V

def idct(X, norm="ortho"):
    """
    The inverse to DCT-II, which is a scaled Discrete Cosine Transform, Type III
    Our definition of idct is that idct(dct(x)) == x
    For the meaning of the parameter `norm`, see:
    https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.fftpack.dct.html
    :param X: the input signal
    :param norm: the normalization, None or 'ortho'
    :return: the inverse DCT-II of the signal over the last dimension
    """

    x_shape = X.shape
    N = x_shape[-1]

    X_v = X.reshape(-1, x_shape[-1]) / 2

    if norm == "ortho":
        X_v[:, 0] *= np.sqrt(N) * 2
        X_v[:, 1:] *= np.sqrt(N / 2) * 2

    k = (
        torch.arange(x_shape[-1], dtype=X.dtype, device=X.device)[None, :]
        * np.pi
        / (2 * N)
    )
    W_r = torch.cos(k)
    W_i = torch.sin(k)

    V_t_r = X_v
    V_t_i = torch.cat([X_v[:, :1] * 0, -X_v.flip([1])[:, :-1]], dim=1)

    V_r = V_t_r * W_r - V_t_i * W_i
    V_i = V_t_r * W_i + V_t_i * W_r

    V = torch.cat([V_r.unsqueeze(2), V_i.unsqueeze(2)], dim=2)

    v = torch.fft.irfft(torch.view_as_complex(V), n=V.shape[1], dim=1)
    x = v.new_zeros(v.shape)
    x[:, ::2] += v[:, : N - (N // 2)]
    x[:, 1::2] += v.flip([1])[:, : N // 2]

    return x.view(*x_shape)

def dct_2d(x, norm="ortho"):
    """
    2-dimentional Discrete Cosine Transform, Type II (a.k.a. the DCT)
    For the meaning of the parameter `norm`, see:
    https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.fftpack.dct.html
    :param x: the input signal
    :param norm: the normalization, None or 'ortho'
    :return: the DCT-II of the signal over the last 2 dimensions
    """
    X1 = dct(x - 128, norm=norm)
    X2 = dct(X1.transpose(-1, -2), norm=norm)
    return X2.transpose(-1, -2)

def idct_2d(X, norm="ortho"):
    """
    The inverse to 2D DCT-II, which is a scaled Discrete Cosine Transform, Type III
    Our definition of idct is that idct_2d(dct_2d(x)) == x
    For the meaning of the parameter `norm`, see:
    https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.fftpack.dct.html
    :param X: the input signal
    :param norm: the normalization, None or 'ortho'
    :return: the DCT-II of the signal over the last 2 dimensions
    """
    x1 = idct(X, norm=norm)
    x2 = idct(x1.transpose(-1, -2), norm=norm)
    return x2.transpose(-1, -2) + 128

# 3. Block splitting
# From https://stackoverflow.com/questions/41564321/split-image-tensor-into-small-patches
def image_to_patches(image):
    """Splits an image into 8 by 8 patches, each of which has the jpeg compression applied seperately"""
    k = 8
    batch_size, num_channels, height, width = image.shape

    image_reshaped = image.view(batch_size, num_channels, height // k, k, width // k, k)
    image_transposed = torch.transpose(image_reshaped, 3, 4)
    return image_transposed.reshape(batch_size, num_channels, -1, k, k)

# -3. Block joining
def patches_to_image(patches, image_shape):
    """Reverses the process of splitting an image into patches"""

    k = 8
    batch_size, num_channels, height, width = image_shape
    image_reshaped = patches.view(
        batch_size, num_channels, height // k, width // k, k, k
    )
    image_transposed = torch.transpose(image_reshaped, 3, 4)

    return image_transposed.reshape(batch_size, num_channels, height, width)

def ycbcr_to_rgb_jpeg(image):
    """Converts a batch of YCbCr image tensors to RGB image tensor, using the jpeg conversion matrix"""

    matrix_yuv = torch.tensor(
        np.array(
            [[1.0, 0.0, 1.402], [1, -0.344136, -0.714136], [1, 1.772, 0]],
            dtype=np.float32,
        ).T,
        device=config.device,
    )

    shift_yuv = torch.FloatTensor([0, -128, -128]).to(config.device)

    image = image.permute(0, 2, 3, 1)
    image = torch.tensordot(image + shift_yuv, matrix_yuv, dims=1)
    image = image.permute(0, 3, 1, 2)

    return image

def rgb_to_ycbcr_jpeg(image):
    """Converts a batch of RGB image tensors to YCbCr image tensors"""

    matrix_rgb = torch.as_tensor(
        np.array(
            [
                [0.299, 0.587, 0.114],
                [-0.168736, -0.331264, 0.5],
                [0.5, -0.418688, -0.081312],
            ],
            dtype=np.float32,
        ).T,
        device=config.device,
    )

    shift_rgb = torch.as_tensor([0.0, 128.0, 128.0], device=config.device)

    image = image.permute(0, 2, 3, 1)
    image = torch.tensordot(image, matrix_rgb, dims=1) + shift_rgb
    image = image.permute(0, 3, 1, 2)

    return image

def jpeg_encode(image, noise_vars):
    """Applies JPEG compression to an image, and adds the noise variables to the quantized coefficients"""

    # Whole algorithm functions with the assumption images are in the range [0, 255], so transform to this range
    noise_vars = noise_vars * 255
    image = image * 255

    # "Compression"
    # Change to the ycbr colour space, which is more suited for compression due to the human eye's sensitivity to luminance (which is concentrated in  the Y channel)
    image = rgb_to_ycbcr_jpeg(image)
    comp = image_to_patches(image)  # Split the image into 8x8 blocks
    # Apply the dicrete cosine transform to each block, changing it from the spatial domain to the frequency domain
    comp = dct_2d(comp)

    comp = (
        comp + noise_vars
    )  # Add the noise variables to the frequency domain coefficients

    return comp

def jpeg_decode(components, image_shape):
    """
    Transforms from a jpeg compressed image to a normal image.
    """
    comp = idct_2d(components)
    image = patches_to_image(comp, image_shape)
    image = ycbcr_to_rgb_jpeg(image)

    image = torch.clamp(image / 255.0, 0, 1)

    return image

def apply_jpeg(image, quantiz_err_vars):
    """
    Applies JPEG compression to an image, and adds the noise variables to the quantized coefficients.
    """
    jpeg_image = jpeg_encode(image, quantiz_err_vars)
    rgb_image = jpeg_decode(jpeg_image, image.shape)
    return rgb_image

class JPEGAdversary(nn.Module):
    """
    Implements the JPEG attack, which works by transforming an image into the JPEG domain,
    adding noise to the quantization error, and then transforming it back into the RGB domain. Has been
    edited from the original implementation in https://arxiv.org/pdf/1908.08016.pdf.

    Parameters
    ---

    num_steps : int
        The number of steps to take in the optimization process.

    step_size : float
        The step size to take in the optimization process.

    epislon: float
        The maximum amount of noise to add to the quantization error.

    """

    def __init__(self, epsilon, num_steps, step_size):
        super().__init__()
        self.num_steps = num_steps
        self.step_size = step_size
        self.epsilon = epsilon

    def forward(self, model, inputs, targets):
        batch_size, num_channels, height, width = inputs.size()

        if height % 8 != 0 or width % 8 != 0:
            raise Exception

        noise_vars = self.epsilon * torch.rand(
            (batch_size, num_channels, height // 8 * width // 8, 8, 8)
        ).to(config.device)
        noise_vars.requires_grad = True

        for _ in range(self.num_steps):
            adv_inputs = apply_jpeg(inputs.detach(), noise_vars)

            outputs = model(adv_inputs)
            loss = F.cross_entropy(outputs, targets)

            grads = torch.autograd.grad(loss, noise_vars)[0]

            noise_vars = noise_vars + self.step_size * torch.sign(grads)
            noise_vars = torch.clamp(noise_vars, -self.epsilon, self.epsilon)

        return apply_jpeg(inputs.detach(), noise_vars)

class JPEGAttack(AttackInstance):
    def __init__(self, model, args):
        super().__init__(model, args)
        self.attack = JPEGAdversary(
            epsilon=args.epsilon, num_steps=args.num_steps, step_size=args.step_size
        )

    def generate_attack(self, batch):
        xs, ys = batch
        return self.attack(self.model, xs, ys)

def get_attack(model, args):
    return JPEGAttack(model, args)


import openai
import json
import ast
from sklearn.model_selection import train_test_split
import os
import time

def main():
    openai.api_key = open("ID/my_api_key.txt").read().strip()

    data_list = create_data_list("data/original_data.json")
    training_question_list = []
    testing_question_list = []

    for data in data_list:
        qna_string = get_questions(data)
        if qna_string == -1:
            continue
        try:
            qna_list = extract_qa(qna_string)
            if len(qna_list) == 1:
                training_question_list += qna_list
            else:
                training_question_list += qna_list[:-1]
                testing_question_list.append(qna_list[-1])
            print(training_question_list)
            print(testing_question_list)
        except:
            continue

    model_ID = train(training_question_list, testing_question_list)
    #model_ID = create_finetune_model(model='curie', batch_size=8, n_epochs=20, training_file="data/training_data.jsonl", validation_file="data/testing_data.jsonl")
    model_ID_file = "ID/my_fine-tuned_model_ID.txt"
    with open(model_ID_file, "w") as f:
        f.write(model_ID)
    f.close()

    return 0

def create_data_list(original_json_file):
    f = open(original_json_file)
    data = json.load(f)
    data_list = []
    for title in data:
        for command in data[title]:
            description = data[title][command]
            data_list.append([title, command, description])
    f.close()
    print(data_list)
    return data_list

def get_questions(data):
    category, command, function = data[0], data[1], data[2]
    context = f"{category}: {command}\nFunction: {function}"
    content = f"Please write at least five question-answer pairs based on the text below, in the format of a nested python list. Each inner list should contain one question and one answer.\n\n{context}"
    example_context = "command: p4 add\nFunction: to add new files"
    example_content = f"Example:\nPlease write at least five question-answer pairs based on the text below, in the format of a nested python list. Each inner list should contain one question and one answer.\n\n{example_context}"

    try:
        response = openai.ChatCompletion.create(
            model="gpt-3.5-turbo",
            messages=[
                {"role": "system", "content": "You are good at generating question-answer pairs based on a given text."},
                {"role": "user", "content": example_content},
                {"role": "assistant", "content": "qa_list = [\n['What is the p4 command to add a new file?', 'p4 add'],\n['What is the purpose of p4 add?', 'to add a new file'],\n['When do you use p4 add?', 'when you want to add a new file'],\n['What command should you use when you want to add a new file?', 'p4 add'],\n['What is the function of p4 add?','to add new files']\n]"},
                {"role": "user", "content": content}
            ]
        )
    except openai.error.ServiceUnavailableError:
        return -1

    msg = response['choices'][0]['message']['content']

    print("This is the response:\n")
    print(msg)
    return msg

def extract_qa(data_string):
    if '->' in data_string:
        data_string = data_string.replace('->', 'implies')
    if '\n' in data_string:
        data_string = data_string.replace('\n', ' ')

    # Extract the content inside the square brackets using literal_eval
    qa_pairs = ast.literal_eval(data_string.split('=')[-1])
    print(qa_pairs)

    return qa_pairs

# train model, returns the ID of the trained model
def train(train_df, test_df):
    print(train_df)
    print(test_df)
    with open("data/training_data.jsonl", "w") as f:
        for qa_pair in train_df:
            prompt_completion_pair = {"prompt": qa_pair[0] + " ->", "completion": " " + qa_pair[1] + "\n"}
            json.dump(prompt_completion_pair, f)
            f.write("\n")
    f.close()
    with open("data/testing_data.jsonl", "w") as f:
        for qa_pair in test_df:
            prompt_completion_pair = {"prompt": qa_pair[0] + " ->", "completion": " " + qa_pair[1] + "\n"}
            json.dump(prompt_completion_pair, f)
            f.write("\n")
    f.close()

    fine_tuned_model_id = create_finetune_model(model='curie', batch_size=len(train_df), n_epochs=20, training_file="data/training_data.jsonl", validation_file="data/testing_data.jsonl")
    return fine_tuned_model_id

def create_file(file_name):
    upload_response = openai.File.create(
        file=open(file_name, "rb"),
        purpose='fine-tune'
    )
    file_id = upload_response.id
    return file_id

def create_finetune_model(model, batch_size, n_epochs, training_file, validation_file):
    fine_tune_response = openai.FineTune.create(
        model=model,
        batch_size=batch_size,
        n_epochs=n_epochs,
        training_file=create_file(training_file),
        validation_file=create_file(validation_file)
    )

    job_id = fine_tune_response["id"]
    print(f"Fine_tuning job created with ID: {job_id}")

    return job_id

if __name__ == "__main__":
    main()


from django.urls import path
from . import views
from django.conf.urls.static import static
from django.conf import settings

app_name = 'toko'

urlpatterns = [
     path('', views.HomeListView.as_view(), name = 'home'),
     path('product/<slug>/', views.ProductDetailView.as_view(), name='produk-detail'),
     path('checkout/', views.CheckoutView.as_view(), name='checkout'),
     path('add-to-cart/<slug>/', views.add_to_cart, name='add-to-cart'),
     path('remove_from_cart/<slug>/', views.remove_from_cart, name='remove-from-cart'),
     path('order-summary/', views.OrderSummaryView.as_view(), name='order-summary'),
     path('payment/<payment_method>', views.PaymentView.as_view(), name='payment'),
     path('paypal-return/', views.paypal_return, name='paypal-return'),
     path('paypal-cancel/', views.paypal_cancel, name='paypal-cancel'),
     path('about', views.about, name = "about"),
     path('contact', views.contact, name = "contact"),
     path('faq', views.faq, name = "faq"),
]

if settings.DEBUG:
    urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)
    urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)


from __future__ import (absolute_import, division, print_function,
                        unicode_literals) # python 2 as python 3
import numpy as np
from astropy.table import Table#, Column
from astropy.time import Time, TimeDelta
from astropy.units import Quantity

# create an astrotable with a column of integers
n_obs = 10 # number of rows
n_obs_start = 1 # start counting at 1
astro_table = Table()
obs_id = np.arange(n_obs_start, n_obs_start + n_obs)
astro_table['OBS_ID'] = obs_id

# try Time and TimeDelta columns

# start time
# random points between the start of 2010 and the end of 2014
datestart = Time('2010-01-01 00:00:00', format='iso', scale='utc')
dateend = Time('2015-01-01 00:00:00', format='iso', scale='utc')
time_start = Time((dateend.mjd - datestart.mjd)*np.random.random(len(obs_id)) + datestart.mjd, format='mjd', scale='utc').iso
astro_table['TSTART'] = time_start

# on time: 30 min
ontime = TimeDelta(30.*60.*np.ones_like(obs_id), format='sec')
#col_ontime = Column(name='ONTIME', data=ontime) #faulty!
#astro_table.add_column(col_ontime)
astro_table['ONTIME'] = ontime #works fine, but still no unit showing in the column
# ref: https://github.com/astropy/astropy/issues/3832

# try Quantity instead of TimeDelta quantity

# livetime: 25 min
livetime = Quantity(25.*60.*np.ones_like(obs_id), 'second')
astro_table['LIVETIME'] = livetime

print("astro_table")
print(astro_table)

print()

print("type time col", type(astro_table['TSTART']))
print("type timedelta col", type(astro_table['ONTIME']))
print("type quantity col", type(astro_table['LIVETIME']))

# read columns from the table
#import IPython; IPython.embed() # before error line
time_col = Time(astro_table['TSTART']) # works fine
timedelta_col = TimeDelta(astro_table['ONTIME']) # used to give an error
quantity_col = TimeDelta(astro_table['LIVETIME']) # works fine

print()

#print(time_col.__dict__)
#print(timedelta_col.__dict__) # used to be faulty
#print(quantity_col.__dict__)

print(repr(time_col))
print(repr(timedelta_col)) # used to be faulty
print(repr(quantity_col))


import cv2
import imutils
import numpy as np
import pytesseract

# select video device
cap = cv2.VideoCapture(0)

while True:
    _, frame = cap.read()

    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

    # apply filter and localization
    bfilter = cv2.bilateralFilter(gray, 11, 17, 17)  # Noise reduction
    edged = cv2.Canny(bfilter, 30, 200)  # Edge detection

    # find contours and apply mask
    keypoints = cv2.findContours(edged.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
    contours = imutils.grab_contours(keypoints)
    contours = sorted(contours, key=cv2.contourArea, reverse=True)[:10]

    location = None
    for contour in contours:
        approx = cv2.approxPolyDP(contour, 10, True)
        if len(approx) == 4:
            location = approx
            break
    # loop over our contours
    for contour in contours:
        # approximate the contour
        peri = cv2.arcLength(contour, True)
        approx = cv2.approxPolyDP(contour, 0.018 * peri, True)

        # if our approximated contour has four points, then
        # we can assume that we have found our screen
        if len(approx) == 4:
            location = approx
            break

    if location is None:
        detected = 0
        # print("No contour detected")
    else:
        detected = 1

    if detected == 1:
        cv2.drawContours(frame, [location], -1, (0, 255, 0), 3)

        mask = np.zeros(gray.shape, np.uint8)
        new_image = cv2.drawContours(mask, [location], 0, 255, 255)
        new_image = cv2.bitwise_and(frame, frame, mask=mask)

        # Now crop
        (x, y) = np.where(mask == 255)
        (topx, topy) = (np.min(x), np.min(y))
        (bottomx, bottomy) = (np.max(x), np.max(y))
        Cropped = gray[topx:bottomx + 1, topy:bottomy + 1]

        # Read the number plate
        text = pytesseract.image_to_string(Cropped, config='--psm 11')
        print("Detected Number is:", text)

    cv2.imshow("Camera", frame)

    if cv2.waitKey(1) == ord('q'):
        break

cap.release()
cv2.destroyAllWindows()


import argparse
import warnings

import termcolor
import torch

import models as model_module
import datasets as dataset_module
import models.loss as loss_module
import optimizers as optim_module
from trainer.trainer import Trainer
from torch.nn import DataParallel
import wandb
from config_parser import ConfigParser
from utils.utils import text_collate

parser = argparse.ArgumentParser()
parser.add_argument("--resume", default=False, action="store_true", help="resume training?")
parser.add_argument("--config", default="./config/bart.json", type=str, help="path to config file")
args = parser.parse_args()

def train(args):
    config = ConfigParser(args)

    device = 'cuda' if torch.cuda.is_available() else 'cpu'
    model = config.init_obj(model_module, "arch")
    if config['arch']['type'] == 'LSTMSentimentAnalysis':
        tokenizer = None
        collate_fn = None
    else:
        tokenizer = model.tokenizer
        collate_fn = lambda batch: text_collate(batch, tokenizer, device, max_seq_length=config['max_seq_length'])
    model = DataParallel(model).to(device)

    print(f"Model: {config['name']}, num params: {sum(p.numel() for p in model.parameters())}")

    train_set = config.init_obj(dataset_module, "train_set")
    test_set = config.init_obj(dataset_module, "test_set")

    train_loader = config.init_obj(torch.utils.data, "train_loader", dataset=train_set, collate_fn=collate_fn)
    test_loader = config.init_obj(torch.utils.data, "test_loader", dataset=test_set, collate_fn=collate_fn)

    optimizer = config.init_obj(optim_module, "optimizer", model.parameters())
    if isinstance(optimizer, optim_module.SAM):
        lr_scheduler = optim_module.get_lr_scheduler(optimizer.base_optimizer, config, train_loader)
    else:
        lr_scheduler = optim_module.get_lr_scheduler(optimizer, config, train_loader)
    criterion = config.init_obj(loss_module, "criterion")

    trainer = Trainer(model=model,
                      config=config,
                      criterion=criterion,
                      optimizer=optimizer,
                      device=device,
                      train_loader=train_loader,
                      test_loader=test_loader,
                      lr_scheduler=lr_scheduler,
                      logger=True)
    trainer.train(resume=args.resume)

if __name__ == "__main__":
    wandb.init(project="BART-sentiment-analysis")
    warnings.filterwarnings('ignore')
    train(args)


import os
import sys
import numpy as np

sys.path.append(f"JPG_Decoder")
from JPG_Decoder.jpg_decoder import JPGImage

import pickle
import base64
import zlib

def pack(v):
    return zlib.compress(base64.b64encode(pickle.dumps(v)))

def unpack(v):
    return pickle.loads(base64.b64decode(zlib.decompress(v)))

fpath="cropped"
imgl=os.listdir(fpath)
# out_path="DCresults"

def getDC(file):
    with open(file,"rb") as f:
        jpg=JPGImage(f)
        jpg.read_header()
        jpg.read_data()
        jpg.decode_huffman_data()
        return jpg.DC
result={}
for img in imgl:
    if img.endswith(".jpg") and "Green" in img:
        dc=getDC(os.path.join(fpath,img))
        result[img]=dc
        print(img)

with open("Button_results","wb") as f:
    f.write(pack(result))


import pyttsx3
import speech_recognition as sr
import datetime
import wikipedia
import os

import webbrowser

engine = pyttsx3.init('sapi5')
voices = engine.getProperty('voices')
#print(voices[0].id)
engine.setProperty('voice',voices[1].id)

def wishMe():
    hour = int(datetime.datetime.now().hour)
    if hour>=0 and hour<12:
        speak("Good Morning")
    elif hour>=12 and hour<18:
        speak("Good Afternoon")
    else:
        speak("Good Evening")
    speak(" Sir Im Jarvis")

def speak(audio):
    engine.say(audio)
    engine.runAndWait()

def takeCommand():
    r=sr.Recognizer()
    with sr.Microphone() as source:
        print("hearing....")
        r.pause_threshold = 1
        audio = r.listen(source)

    try:
        print("recognizing")
        query = r.recognize_google(audio, language = 'en-in')
        print(f"user said: {query}\n")

    except Exception as e:
        # print(e) hiding error

        print("can you please speak again sir")
        return "None"
        #return string none
    return query

if __name__ == "__main__":

    wishMe()
    speak("yash is a good f boy")
    print("hi")
    query = takeCommand().lower()

    if 'wikepedia' in query:
        speak("searching wiki")
        query = query.replace("wikipedia","")
        results = wikipedia.summary(query, sentences=2)
        speak("according to wiki")
        print(results)
        speak(results)

    elif 'open' in query:
        if 'youtube' in query:
            webbrowser.open("youtube.com")
        elif 'google' in query:
            webbrowser.open("google.com")
        elif 'facebook' in query:
            webbrowser.open("fb.com")
        elif 'instagram' in query:
            webbrowser.open("instagram.com")
        elif 'facebook' in query:
            webbrowser.open("fb.com")


import json

import graphene
from graphene_django import DjangoObjectType
from graphene_django.filter import DjangoFilterConnectionField

from helpers.constants import EventStatus
from helpers.utils import enum_to_dict
from services.authentication import GraphqlAuthentication

from .models import Event, FantasyPlayer, Marblelympics, PlayerEntry, Team

class TeamNode(DjangoObjectType):
    class Meta:
        model = Team
        interfaces = (graphene.relay.Node,)
        exclude_fields = ('image')
        filter_fields = {
            'name': ['exact', 'icontains', 'istartswith'],
            'ml_participated': ['exact'],
        }

class MLNode(DjangoObjectType):
    player_count = graphene.Int()

    class Meta:
        model = Marblelympics
        interfaces = (graphene.relay.Node,)
        filter_fields = {
            'active': ['exact'],
        }

    def resolve_player_count(self, info, **kwargs):
        return self.player_count

class SelectionNode(graphene.ObjectType):
    position = graphene.Int()
    team = graphene.Field(TeamNode)

class EventNode(DjangoObjectType):
    status = graphene.Int()
    entry_count = graphene.Int()
    results = graphene.List(SelectionNode)

    class Meta:
        model = Event
        interfaces = (graphene.relay.Node,)
        exclude_fields = ('scoring_status')
        filter_fields = {
            'ml__active': ['exact'],
        }

    def resolve_entry_count(self, info, **kwargs):
        return self.entry_count

    def resolve_results(self, info, **kwargs):
        results = json.loads(self.results)
        return [
            SelectionNode(
                position=int(position),
                team=Team.objects.get(name=name),
            )
            for position, name in results.items()
        ]

class FantasyPlayerNode(DjangoObjectType):
    class Meta:
        model = FantasyPlayer
        interfaces = (graphene.relay.Node,)
        filter_fields = {
            'marblelympics__active': ['exact'],
        }

class PlayerEntryNode(DjangoObjectType):
    selections = graphene.List(SelectionNode)

    class Meta:
        model = PlayerEntry
        interfaces = (graphene.relay.Node,)
        filter_fields = {
            'player__marblelympics__active': ['exact'],
        }

    def resolve_selections(self, info, **kwargs):
        selections = json.loads(self.selections)
        return [
            SelectionNode(
                position=int(position),
                team=Team.objects.get(name=name),
            )
            for position, name in selections.items()
        ]

class Query(graphene.ObjectType):
    team = graphene.relay.Node.Field(TeamNode)
    teams = DjangoFilterConnectionField(TeamNode)
    marblelympics = DjangoFilterConnectionField(MLNode)
    event = graphene.relay.Node.Field(EventNode)
    events = DjangoFilterConnectionField(EventNode)
    fplayer = graphene.relay.Node.Field(FantasyPlayerNode)
    fplayers = DjangoFilterConnectionField(FantasyPlayerNode)
    fplayer_entry = graphene.relay.Node.Field(PlayerEntryNode)
    fplayer_entries = DjangoFilterConnectionField(PlayerEntryNode)

class UpsertPlayerEntry(graphene.relay.ClientIDMutation):
    entry = graphene.Field(PlayerEntryNode)

    class Input:
        event_number = graphene.Int()
        team_1 = graphene.String()
        team_2 = graphene.String()
        team_3 = graphene.String()

    def mutate_and_get_payload(self, info, **input):
        user = GraphqlAuthentication.get_user(self, info, **input)
        player = user.player

        ml = Marblelympics.objects.filter(active=True).first()
        if not ml:
            raise Exception("No active ML")

        event = Event.objects.filter(
            ml=ml,
            number=input.get('event_number'),
        ).first()
        if not event:
            raise Exception(f"No event #{input.get('event_number')} in {ml}")
        if enum_to_dict(EventStatus).get(event.status, 1) == 'LOCKED':
            raise Exception(f"Event is locked")

        fplayer = FantasyPlayer.objects.filter(
            player=player,
            marblelympics=ml,
        ).first()
        if not fplayer:
            fplayer = FantasyPlayer.objects.create(
                player=player,
                marblelympics=ml,
            )

        entry = PlayerEntry.objects.filter(
            event=event, player=fplayer
        ).first() or PlayerEntry()

        entry.event = event
        entry.player = fplayer
        entry.selections = json.dumps(
            {
                '1': input.get('team_1') or '',
                '2': input.get('team_2') or '',
                '3': input.get('team_3') or '',
            }
        )
        entry.save()

        return UpsertPlayerEntry(entry=entry)

class Mutation(graphene.AbstractType):
    upsert_player_entry = UpsertPlayerEntry.Field()


import logging
import pandas as pd

log = logging.getLogger(__name__)

class CustomData(object):
    """
    This class I/O's custom case data

    INPUT: tab-delimited text

    ATTRIBUTES:
    .dat -- pd DataFrame from input text. row index = 'case_number'
    """
    def __init__(self, fp):
        self.dat = self.__load(fp)

    def __load(self, filepath):
        a = pd.read_csv(filepath, sep="\t")

        # ### Handle column names ###
        a.columns = [c.lower() for c in a.columns.values]
        a_cols = a.columns.tolist()

        # Convert 'CASE*' => 'case_number'
        case_index = next((index for index, value in enumerate(a_cols)
                           if value[:4] == 'case'), None)
        if case_index is not None:
            a_cols[case_index] = 'case_number'

        a.columns = a_cols

        a.set_index(keys='case_number', drop=True, inplace=True)
        return a


import os

import openpyxl as openpyxl

class ExcelUtil:
    # 获取项目路径
    def get_object_path(self):
        print(os.path.abspath(os.path.dirname(__file__)).split("common")[0])

    def read_execl(self):
        # openpyxl,xlrd  python中读取execl的第三方服务
        # 加载execl工作簿
        wb = openpyxl.load_workbook(self.get_object_path() + "execl存放路径")
        # 获得sheet对象
        sheet = wb['sheet名称']
        # 获得execl的行数和列数
        print(sheet.max_row, sheet.max_column)
        # 循环
        # for rows in range(1, sheet.max_row + 1):
        #     for cols in range(1, sheet.max_column + 1):
        #         print(rows, cols)  # 显示可读取的行和列数量
        #         print(sheet.cell(rows, cols).value)  # 显示行和列的数据
        # 把以上数据组成需要的形式
        all_list = []
        for rows in range(1, sheet.max_row + 1):
            temp_list = []
            for cols in range(1, sheet.max_column + 1):
                temp_list.append(sheet.cell(rows, cols).value)
            all_list.append(temp_list)
        return all_list


import os

from kivy.uix.boxlayout import BoxLayout
from kivy.uix.floatlayout import FloatLayout
from kivy.properties import ObjectProperty
from kivy.uix.popup import Popup
import csv

class LoadDialog(FloatLayout):
    load = ObjectProperty(None)
    cancel = ObjectProperty(None)
    current_dir = os.path.dirname(os.path.abspath(__file__))

class ButtonZone(BoxLayout):
    id_name = "buttonzone"

    loadfile = ObjectProperty(None)
    text_input = ObjectProperty(None)

    def dismiss_popup(self):
        self._popup.dismiss()

    def show_load(self):
        content = LoadDialog(load=self.load, cancel=self.dismiss_popup)
        self._popup = Popup(title="Load File", content=content, size_hint=(0.9, 0.9))
        self._popup.open()

    def load(self, path, filename):
        workerinfos = []
        with open(os.path.join(path, filename[0]), "r") as f:
            reader = csv.reader(f)
            header = next(reader)

            for data in reader:
                if data[2] == "JPY":
                    data[1] = int(data[1])
                else:
                    data[1] = float(data[1])
                workerinfos.append(data)

        self.parent.send_csvinfo_to_stack(workerinfos)
        self.dismiss_popup()

    def clicked_add(self):
        name = self.ids["text_name"].text
        wag = self.ids["text_wag"].text
        typ = self.ids["text_type"].text

        if name == "" or wag == "":
            return

        self.ids["text_wag"].text = ""

        if typ == "JPY":
            try:
                wag = int(wag)
            except ValueError:
                print("ValueError")
                return
        else:
            wag = float(wag)

        self.ids["text_name"].text = ""

        print(name, wag)
        self.parent.send_info_to_stack([name, wag, typ])

    def allsum(self):
        print("allsum")
        self.parent.send_info_to_cmd()

    def resetcmd(self):
        self.parent.reset_cmd()


import pygame.font

class BUtton:
    def __init__(self, ai_game, msg):
        """初始化按钮属性"""
        # msg是要在按钮上显示的文本
        self.screen = ai_game.screen
        self.screen_rect = ai_game.screen.get_rect()

        # 设置按钮的尺寸和其他属性
        self.width, self.height = 200, 50
        self.buttom_color=(0,255,0)
        self.text_color=(255,255,255)
        self.font=pygame.font.SysFont(None,48)

        #创建按钮的rect对象，并使其居中
        self.rect=pygame.Rect(0,0,self.width,self.height)
        self.rect.center=self.screen_rect.center

        #按钮的标签只创建一次
        self._pre_msg(msg)

    def _pre_msg(self,msg):
        """将msg渲染为图像,并使其在按钮上居中"""
        self.msg_image=self.font.render(msg,True,self.text_color
                                        ,self.buttom_color)
        #布尔实参指定开启还是关闭反锯齿功能，反锯齿让文本的边缘更平滑
        self.msg_image_rect=self.msg_image.get_rect()
        #让图像文本在按钮上居中
        self.msg_image_rect.center=self.rect.center

    def draw_button(self):
        """绘制一个用颜色填充的按钮，再绘制文本"""
        self.screen.fill(self.buttom_color,self.rect)
        self.screen.blit(self.msg_image,self.msg_image_rect)


#!/usr/bin/env python3

import yaml

from slack_cleaner2 import *

with open("config.yml", "r") as configfile:
  cfg = yaml.load(configfile, Loader=yaml.BaseLoader)

f = is_not_pinned()
before = a_while_ago(days=7)

for token in cfg["tokens"]:
  s = SlackCleaner(token)
  s.sleep_for = 1.0

  with s.log.group("files"):
    for entry in filter(f, s.files(before=before)):
      entry.delete()

  for c in s.conversations:
    with s.log.group(c.name):
      for msg in filter(f, c.msgs(before=before)):
        if (c.type.value != 4) or (msg.user == s.myself):
          msg.delete(as_user=True, files=True, replies=True)

  s.log.summary()


import re
import time
import cv2
import calendar

gfile = "UMgcode/square.gcode"

#with open(gfile) as gcode:
	#print(gfile)
	#for line in gcode:
		#line = line.strip()
		#layerend = re.findall('LAYER:', line)
		#if layerend:
			#layernum = re.findall(r'-?\d+', line)
			#print(layernum[0])

def find_init_temperature(gfile):
	extruder_temp = 0
	bed_temp = 0
	with open(gfile) as gcode:
		for line in gcode:
			#print(line)
			line = line.strip()
			extemp = re.findall(";EXTRUDER_TRAIN.0.INITIAL_TEMPERATURE:", line)
			btemp = re.findall(";BUILD_PLATE.INITIAL_TEMPERATURE:", line)
			if extemp:
				extruder_temp = line.split(":",1)[1]
				print(extruder_temp)
				flag0 = 1
			if btemp:
				bed_temp = line.split(":",1)[1]
				print(bed_temp)
				flag1 = 1
			if extruder_temp and bed_temp:
				print("here")
				return extruder_temp, bed_temp
				break

#temp0, temp1 = find_init_temperature(gfile)

#print(temp0)

###############################################
#file0 = open(gfile, 'r')
#count = 0
#while True:
	#try:
		#time.sleep(.0001)
		#count += 1
		#print("\n\n"+str(count)+"\n\n")

		#line = file0.readline()
		#print(line)
		#if not line:
		#	break
	#except(KeyboardInterrupt):
		#print("\nExit program.\n")
		#break

################################################
#times = open("times.txt","w+")

#count = 1
#time1 = 1
#with open(gfile) as gcode:
#	print(gfile)
#	for line in gcode:
#		line = line.strip()
#		timeline = re.findall(';TIME_ELAPSED:', line)
#		if timeline:
#			time2 = re.findall(r'[\d.\d]+', line)
#			time2 = float(time2[0])
#			time_elapsed = time2 - time1
#			times.write(str(time_elapsed)+","+str(count)+"\n")
#			print(time_elapsed)
#			time1 += 1
#			print(count)
#		count += 1

def get_time_elapsed(times_file):
	line = times_file.readline()
	items = line.split(",")
	elapsed_time = re.findall(r'[\d.\d]+', items[0])
	time_line = re.findall(r'[\d.\d]+', items[1])
	elapsed_time = float(elapsed_time[0])
	time_line = int(time_line[0])
	return elapsed_time, time_line

def capture_img(camera):
	cam_fps = camera.get(cv2.CAP_PROP_FPS)
	print('Capture Image at %.2f FPS.' %cam_fps)
	ts = calendar.timegm(time.gmtime())
	imfile = str(ts)+'img.jpg'
	print(imfile)
	cv2.imwrite(filename=imfile, img=frame)
	print("Image saved!")

def video_capture(webcam):
	count = 1
	t = True
	while t:
		try:
			count += 1
			print("\n\n"+str(count)+"\n\n")
			check, frame = webcam.read()
			#cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
			cv2.imshow("Capturing", frame)
			key = cv2.waitKey(1)
			if count == 50:
				capture_img(webcam)
				t = False
				break
		except(KeyboardInterrupt):
			cv2.destroyAllWindows()
			break

webcam = cv2.VideoCapture(0)
webcam.set(cv2.CAP_PROP_FRAME_WIDTH, 800)#640)
webcam.set(cv2.CAP_PROP_FRAME_HEIGHT, 600)#480)
webcam.set(cv2.CAP_PROP_FPS, 25)#30)

times_file = "times.txt"
linecount = 1
times = open(times_file,"r")

gfile_print = open(gfile, "r")

elapsed_time, layerbreak = get_time_elapsed(times)
print(elapsed_time, layerbreak)

while True:
	try:
		time.sleep(.001)
		line = gfile_print.readline()
		print(linecount)
		check, frame = webcam.read()
		#cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
		cv2.imshow("Capturing", frame)
		key = cv2.waitKey(1)

		if linecount == layerbreak:
			u = input("\nopen video capure\n")
			video_capture(webcam)
			elapsed_time, layerbreak = get_time_elapsed(times)
			print(elapsed_time, layerbreak)
			u = input("\nlayerbreak\n")

		if not line:
			print("\n\nEnd of file\n\n")
			break

		linecount += 1

	except(KeyboardInterrupt):
		webcam.release()
		cv2.destroyAllWindows()
		print("\nExit program.\n")
		break


import gzip
import json
import argparse
from tqdm import tqdm

if __name__=="__main__":
    parser = argparse.ArgumentParser(description='Create bilingual pretraining file for tevatron.')
    parser.add_argument('--input', dest='input', required=True)
    parser.add_argument('--output', dest='output', required=True)

    args = parser.parse_args()

    with open(args.input) as f, open(args.output, "w") as g:
        for line in tqdm(f):
            query, text = line.strip().split("\t")
            query = query.strip().lower()
            text = text.strip().lower()

            temp = {}
            temp["query"] = query
            temp["positives"] = [pos_text]
            temp["negatives"] = []

            g.write(json.dumps(temp)+"\n")


"""
This example is inspired by https://realpython.com/python-gil/ and designed to address issues with performance bottlenecks associated with multithreading when data collection and analysis are done using the same process hence sharing Python Interpretter and are subject to GIL.

"""

from numpy import random

# multi_threaded.py
import time
from threading import Thread
results = [None]*2
def generate_random_array(size = (10,1000,1000), dtype = 'uint16'):
    from numpy import random
    return random.randint(0,4095,size = size).astype(dtype)

def save_to_drive(arr,filename = '1.h5py'):
    from h5py import File
    from tempfile import gettempdir
    with File(gettempdir()+'/'+filename,'a') as f:
        f.create_dataset('data', data = arr, chunks= (1,1000,1000))
arr1 = generate_random_array(size = (100,1000,1000))
arr2 = generate_random_array(size = (100,1000,1000))
arr3 = generate_random_array(size = (100,1000,1000))
# t1 = Thread(target=save_to_drive, args=(arr1,str(time.time())+'_1.h5py'))
# t2 = Thread(target=save_to_drive, args=(arr2,str(time.time())+'_2.h5py'))
# t3 = Thread(target=save_to_drive, args=(arr3,str(time.time())+'_3.h5py'))
# start = time.time()
# t1.start()
# t2.start()
# #t3.start()
# t1.join()
# t2.join()
# #t3.join()
# end = time.time()
#
# print('Time taken in seconds -', end - start)

start = time.time()
save_to_drive(arr1,str(time.time())+'_1.h5py')
save_to_drive(arr2,str(time.time())+'_2.h5py')
#save_to_drive(arr3,str(time.time())+'_3.h5py')
end = time.time()

print('Time taken in seconds -', end - start)


from django.test import TestCase
from django.contrib.gis.db import models
from itertools import chain
from streamwebs.models import CameraPoint, PhotoPoint, Site

class PhotoPointTestCase(TestCase):

    def setUp(self):
        self.expected_fields = {
            'camera_point': models.ForeignKey,
            'number': models.PositiveSmallIntegerField,
            'pp_date': models.DateField,
            'compass_bearing': models.PositiveSmallIntegerField,
            'distance': models.DecimalField,
            'camera_height': models.DecimalField,
            'notes': models.TextField,
            'id': models.AutoField,

            'camera_point_id': models.ForeignKey,
            'photo_point': models.ManyToOneRel
        }

        self.optional_fields = {
            'notes': models.TextField
        }

        site = Site.test_objects.create_site('test site c')
        self.camera_point = CameraPoint.test_objects.create_camera_point(
            site, '2016-07-05', 'POINT(-121.393401 44.061437)')

    def test_fields_exist(self):
        """
        The model's field types and the expected field types listed in setUp
        should match exactly.
        """
        for field, field_type in self.expected_fields.items():
            self.assertEqual(field_type,
                             type(PhotoPoint._meta.get_field(field)))

    def test_no_extra_fields(self):
        """
        The model's field names and the expected field names listed in setUp
        should match exactly.
        """
        # MyModel._meta.get_all_field_names() is deprecated. The following is
        # an alternative, as suggested by the Django docs:
        # https://docs.djangoproject.com/en/1.9/ref/models/meta/
        pp_fields = list(set(chain.from_iterable(
            (field.name, field.attname) if hasattr(field, 'attname') else
            (field.name,) for field in PhotoPoint._meta.get_fields()
            if not (field.many_to_one and field.related_model is None)
            )))
        self.assertEqual(sorted(pp_fields),
                         sorted(self.expected_fields.keys()))

    def test_optional_fields(self):
        """
        The model's optional fields should match the ones listed in setUp.
        """
        for field in self.optional_fields:
            self.assertEqual(
                PhotoPoint._meta.get_field(field).blank, True)

    def test_PhotoPoint_ManyToOneCameraPoint(self):
        """
        A photo point should correspond to a single camera point.
        """
        photo_point = PhotoPoint.test_objects.create_photo_point(
            self.camera_point, '2016-07-06', 140, 4.5, 2.3)

        self.assertEqual(photo_point.camera_point.site.site_name,
                         'test site c')
        self.assertEqual(photo_point.camera_point.letter, 'A')
        self.assertEqual(photo_point.camera_point.cp_date, '2016-07-05')
        self.assertEqual(photo_point.camera_point.location.coords,
                         (-121.393401, 44.061437))

    def test_obj_exists_req_fields(self):
        """
        A photo point should be created successfully when the required fields
        are provided.
        """
        photo_point = PhotoPoint.test_objects.create_photo_point(
            self.camera_point, '2016-07-04', 270, 2.3, 0.5)

        # Required
        self.assertEqual(photo_point.camera_point.site.site_name,
                         'test site c')
        self.assertEqual(photo_point.camera_point.cp_date, '2016-07-05')
        self.assertEqual(photo_point.pp_date, '2016-07-04')
        self.assertEqual(photo_point.compass_bearing, 270)
        self.assertEqual(photo_point.distance, 2.3)
        self.assertEqual(photo_point.camera_height, 0.5)

        # Optional
        self.assertEqual(photo_point.notes, '')

    def test_obj_exists_opt_fields(self):
        """
        A photo point should be created successfully when the required and
        optional fields are provided.
        """
        photo_point = PhotoPoint.test_objects.create_photo_point(
            self.camera_point, '2016-07-02', 1.62, 4.1, 5.3,
            'Notes on photo point for test site c')

        self.assertEqual(photo_point.camera_point.site.site_name,
                         'test site c')
        self.assertEqual(photo_point.camera_point.cp_date, '2016-07-05')
        self.assertEqual(photo_point.pp_date, '2016-07-02')
        self.assertEqual(photo_point.compass_bearing, 1.62)
        self.assertEqual(photo_point.distance, 4.1)
        self.assertEqual(photo_point.camera_height, 5.3)
        self.assertEqual(photo_point.notes,
                         'Notes on photo point for test site c')

    def test_number_assign_for_first_pp(self):
        photo_point = PhotoPoint.test_objects.create_photo_point(
            self.camera_point, '2016-07-04', 270, 2.3, 0.5)
        self.assertEqual(photo_point.number, 1)

    def test_number_assign_for_regular_pp(self):
        photo_point = PhotoPoint.test_objects.create_photo_point(
            self.camera_point, '2016-07-04', 270, 2.3, 0.5)

        self.assertEqual(photo_point.number, 1)

        photo_point_2 = PhotoPoint.test_objects.create_photo_point(
            self.camera_point, '2016-07-04', 270, 2.3, 0.5)

        self.assertEqual(photo_point_2.number, 2)

        photo_point_2.number = 44
        photo_point_2.save()

        self.assertEqual(photo_point_2.number, 44)

        photo_point_3 = PhotoPoint.test_objects.create_photo_point(
            self.camera_point, '2016-07-04', 270, 2.3, 0.5)

        self.assertEqual(photo_point_3.number, 45)

    def test_restart_numbering_for_new_cp(self):
        site = Site.test_objects.create_site('test site c')
        camera_point = CameraPoint.test_objects.create_camera_point(
            site, '2016-07-05', 'POINT(-121.393401 44.061437)')
        camera_point_2 = CameraPoint.test_objects.create_camera_point(
            site, '2016-07-05', 'POINT(-121.393401 44.061437)')
        photo_point = PhotoPoint.test_objects.create_photo_point(
            camera_point, '2016-07-04', 270, 2.3, 0.5)
        photo_point_2 = PhotoPoint.test_objects.create_photo_point(
            camera_point_2, '2016-07-04', 270, 2.3, 0.5)

        self.assertEqual(camera_point.site.site_name,
                         camera_point_2.site.site_name)
        self.assertEqual(camera_point.letter, 'A')
        self.assertEqual(camera_point_2.letter, 'B')
        self.assertEqual(photo_point.number, 1)
        self.assertEqual(photo_point_2.number, 1)


import random
from datetime import datetime

now = datetime.now()

current_time = now.strftime("%H:%M:%S")

def shuffle(arr):
  start = datetime.now()
  l = len(arr)
  a = []
  for i in arr:
    a.insert(random.randrange(l-1), i)
    i += 1
  end = datetime.now()
  total = end-start
  print("shuffle", total)
  return a

def reverse(arr):
  start = datetime.now()
  l = len(arr)
  a = []
  for i in arr:
    a.insert(0, i)
  end = datetime.now()
  total = end-start
  print("reverse", total)
  return a

def sort(arr):
  start = datetime.now()
  for i in range (len(arr) - 1):
    if arr[0] > arr[0+1] == True:
      arr.append(i)
      arr.remove(i)
    else:
      arr.append(i+1)
      arr.remove(i+1)

  if arr[0] > arr[1] == True:
    arr.append(arr[0])
    arr.remove(arr[0])

  end = datetime.now()
  total = end-start
  print("sort", total)
  return arr


from abc import abstractmethod
from audioop import bias

from loguru import logger
import numpy as np
import pandas as pd
import torch
import torch.nn as nn

from src.methods.utils import confidence_interval
from src.utils import set_device, plot_task

from src.adv_train import RCNN, modify_data

class AbstractMetaLearner(nn.Module):
    """
    Abstract class for meta-learning models. Extensions of this class must define the set_forward function.
    """

    def __init__(self,
                 model_func,
                 transportation=None,
                 use_test_query=False,
                 use_train_query=False,
                 training_stats=None,
                 adv_train_config=None,
                 **args
                 ):
        """

        Args:
            model_func (src.backbones object): backbone model initialized
            training_stats (Statistics): training statistics of the model, updated during training

        """
        super(AbstractMetaLearner, self).__init__()
        # importing inside class to avoid cyclic import
        from src.adv_train import transductive_adv_loss

        self.args = args
        self.report_ray = args.get("report_ray", False)
        self.n_episodes = args.get('n_episodes')
        self.feature = model_func
        self.training_stats = training_stats
        self.transportation_module = transportation
        self.use_test_query = use_test_query
        self.use_train_query = use_train_query
        self.loss_fn = nn.CrossEntropyLoss()
        self.adv_train_config = adv_train_config

        if self.use_test_query and (self.adv_train_config is not None):
            self.transductive_loss_fn = transductive_adv_loss
        else:
            self.transductive_loss_fn = None

        if 'ANIL' in args.get("model_name"):
            self.feature.cls = set_device(
                                nn.Linear(self.feature.final_feat_dim, args.get("n_way"), bias=True)
                                )
            self.meta_learner = set_device(self.build_metalearner(args.get("base_lr")))

    @abstractmethod
    def set_forward(self, support_images, support_labels, query_images):
        """
        Predict query set labels using information from support set labelled images.
        Must be implemented in all classes extending AbstractMetaLearner.
        Args:
            support_images (torch.Tensor): shape (number_of_support_set_images, **image_shape)
            support_labels (torch.Tensor): artificial support set labels in range (0, n_way)
            query_images (torch.Tensor): shape (number_of_query_set_images, **image_shape)

        Returns:
            torch.Tensor: shape(n_query*n_way, n_way), classification prediction for each query data
        """
        pass

    def fit_on_task(
        self, support_images, support_labels,
        query_images, query_labels,
        optimizer,
    ):
        """
        Perform a forward pass and a backward pass on one episode.
        Args:
            support_images (torch.Tensor): shape (number_of_support_set_images, **image_shape)
            support_labels (torch.Tensor): artificial support set labels in range (0, n_way)
            query_images (torch.Tensor): shape (number_of_query_set_images, **image_shape)
            query_labels (torch.Tensor): artificial query set labels in range (0, n_way)
            optimizer (torch.optim.Optimizer): model optimizer

        Returns:
            tuple(torch.Tensor, torch.Tensor): detached from the computational graph
                - shape(n_query*n_way, n_way), classification prediction for each query data
                - shape(,), training loss
        """
        optimizer.zero_grad()
        scores = self.set_forward(support_images, support_labels, query_images)
        loss = self.loss_fn(scores, query_labels)
        loss.backward()
        optimizer.step()

        return scores.detach(), loss.detach().item()

    def extract_features(self, support_images, query_images, feature_ext=None):
        """
        Computes the features vectors of the support and query sets
        Args:
            support_images (torch.Tensor): shape (n_support_images, **image_dim) input data
            query_images (torch.Tensor): shape (n_query_images, **image_dim) input data

        Returns:
            Tuple(torch.Tensor, torch.Tensor): features vectors of the support and query sets, respectively of shapes
            (n_support_images, features_dim) and (n_query_images, features_dim)
        """
        # Set to CUDA if available
        support_images = set_device(support_images)
        query_images = set_device(query_images)

        if feature_ext is not None:
            z_support = feature_ext.forward(support_images)
            z_query = feature_ext.forward(query_images)
        else:
            z_support = self.feature.forward(support_images)
            z_query = self.feature.forward(query_images)

        return z_support, z_query

    @staticmethod
    def get_prototypes(features, labels):
        """
        Compute a prototype for each class.
        Args:
            features (torch.Tensor): shape[n_images, feature_dim], feature vectors
            labels (torch.Tensor): shape[n_images], label corresponding to each feature vector
        Returns:
            torch.Tensor: prototypes. shape[number_of_unique_values_in_labels, feature_dim]
        """

        n_way = len(torch.unique(labels))
        # Prototype i is the mean of all instances of features corresponding to labels == i
        return torch.cat(
            [features[torch.nonzero(labels == label)].mean(0) for label in range(n_way)]
        )

    @staticmethod
    def evaluate(scores, query_labels):
        """
        Predict labels of query images and returns the number of correct top1 predictions
        Args:
            scores (torch.Tensor): shape (number_of_query_set_images, n_way)
            query_labels (torch.Tensor): artificial query set labels in range (0, n_way)

        Returns:
            float: classification accuracy in [0,1]
        """

        topk_scores, topk_labels = scores.data.topk(1, 1, True, True)
        top1_correct = float((topk_labels[:, 0] == query_labels).sum())
        return float(top1_correct) / len(query_labels)

    @staticmethod
    def get_task_perf(
        task_id, classification_scores, labels, class_ids, source_id, target_id
    ):
        """
        Records the classification results for each query instance.
        Args:
            task_id (int): index of the task
            classification_scores (torch.Tensor): predicted classification scores
            labels (torch.Tensor): ground truth labels
            class_ids (list[int]): indices of the classes composing the current classification task
            source_id (int): index of the source domain for this task
            target_id (int): index of the target domain for this task

        Returns:
            pd.DataFrame: for each couple (query, class), gives classification score, class_id,
                ground truth query label, current task id and source and target domain,
        """
        return pd.concat(
            [
                pd.DataFrame(
                    {
                        "task_id": task_id,
                        "source_domain": source_id,
                        "target_domain": target_id,
                        "image_id": i,
                        "true_label": int(class_ids[labels[i]]),
                        "predicted_label": [
                            int(class_ids[label])
                            for label in range(classification_scores.shape[1])
                        ],
                        "score": classification_scores[i],
                    }
                )
                for i in range(labels.shape[0])
            ]
        )

    def plot_adv_task(self, adv_tasks, epoch):
        NUM_TASKS_TO_PLOT = 15
        episodes = list(adv_tasks.keys())
        try:
            episodes_to_plot = np.round(np.linspace(0, len(episodes)-1, NUM_TASKS_TO_PLOT)).astype(int)
        except:
            episodes_to_plot = np.round(np.linspace(0, len(episodes)-1, 10)).astype(int)
        for idx in episodes_to_plot:
            data = adv_tasks[episodes[idx]]
            if "query_before" and "query_after" in data:
                key_before = "query_before"
                title_before = "Query - Before Adversarial Projection"
                key_after = "query_after"
                title_after = "Query - After Adversarial Projection"
            elif "support_before" in data and "support_after" in data:
                key_before = "support_before"
                title_before = "Support - Before Adversarial Projection"
                key_after = "support_after"
                title_after = "Support - After Adversarial Projection"

            plot_task(data[key_before][0], data[key_before][1],
                        self.args["plot_path"], epoch, idx, key_before,
                        data["nrow"], data["ncol"], title_before
                        )
            plot_task(data[key_after][0], data[key_after][1],
                        self.args["plot_path"], epoch, idx, key_after,
                        data["nrow"], data["ncol"], title_after
                        )

    def adv_project(self, mode, episode_index, support_images, support_labels, query_images, query_labels, loss_fn):
        adv_tasks = {}
        if self.adv_train_config["{}_project_task".format(mode)] != 0:
            if self.args.get("plot_perturbed", False):
                adv_tasks[episode_index] = {"query_before": (query_images, query_labels),
                                            "nrow" : self.args["n_way"],
                                            "ncol" : self.args["n_target"]
                                            }

            query_images = RCNN(query_images, self.adv_train_config)
            query_images = modify_data(self,
                                    support_images,
                                    support_labels,
                                    query_images,
                                    query_labels,
                                    loss_fn,
                                    self.adv_train_config | {'project_task': self.adv_train_config["{}_project_task".format(mode)]}
                                    )

            if self.args.get("plot_perturbed", False):
                adv_tasks[episode_index]["query_after"] = (query_images, query_labels)

        if self.adv_train_config["{}_project_task".format(mode)] != 1:
            if self.args.get("plot_perturbed", False):
                adv_tasks[episode_index] = {"support_before": (support_images, query_labels),
                                            "nrow" : self.args["n_way"],
                                            "ncol" : self.args["n_source"]
                                            }

            support_images = RCNN(support_images, self.adv_train_config)
            support_images = modify_data(self,
                                        support_images,
                                        support_labels,
                                        query_images,
                                        query_labels,
                                        loss_fn,
                                        self.adv_train_config | {'project_task': self.adv_train_config["{}_project_task".format(mode)]}
                                        )

            if self.args.get("plot_perturbed", False):
                adv_tasks[episode_index]["support_after"] = (support_images, query_labels)

            # update adv learning rate
            if self.adv_train_config.get("adv_lr_sched", None) is not None:
                self.adv_train_config["lr"] = self.adv_train_config["adv_lr_sched"].step(
                                                                self.adv_train_config["lr"]
                                                            )

        return support_images, support_labels, query_images, query_labels, adv_tasks

    def train_loop(self, epoch, train_loader, optimizer):
        """
        Executes one training epoch
        Args:
            epoch (int): current epoch
            train_loader (DataLoader): loader of a given number of episodes
            optimizer (torch.optim.Optimizer): model optimizer

        Returns:
            tuple(float, float): resp. average loss and classification accuracy
        """
        print_freq = self.n_episodes
        adv_tasks = {}
        loss_list = []
        acc_list = []
        for episode_index, (
            support_images,
            support_labels,
            query_images,
            query_labels,
            _,
            _,
            _,
        ) in enumerate(train_loader):
            query_labels = set_device(query_labels)
            support_labels = set_device(support_labels)

            adv_prob = np.random.random()
            if self.use_train_query and \
              (self.adv_train_config is not None) and \
              (adv_prob > 1.0-self.args.get("adv_proj_prob", 0.5)):
                loss_fn = self.loss_fn
                support_images, support_labels, query_images, query_labels, adv_tasks = self.adv_project(
                    'train', episode_index, support_images, support_labels, query_images, query_labels, loss_fn
                )

            self.train()

            scores, loss_value = self.fit_on_task(
                support_images, support_labels,
                query_images, query_labels,
                optimizer,
            )

            loss_list.append(loss_value)

            acc_list.append(self.evaluate(scores, query_labels) * 100)

            if episode_index % print_freq == print_freq - 1:
                logger.info(
                    "Epoch {epoch} | Batch {episode_index}/{n_batches} | Accuracy {acc} | Loss {loss}".format(
                        epoch=epoch,
                        episode_index=episode_index + 1,
                        n_batches=len(train_loader),
                        loss=np.asarray(loss_list).mean(),
                        acc=np.asarray(acc_list).mean()
                    )
                )
        if self.args.get("plot_perturbed", False) and len(adv_tasks):
            self.plot_adv_task(adv_tasks, epoch)
        return np.asarray(loss_list), np.asarray(acc_list)

    def eval_loop(self, test_loader):
        """

        Args:
            test_loader (DataLoader): loader of a given number of episodes

        Returns:
            tuple(float, float, pd.DataFrame): resp. average loss and classification accuracy,
                and advanced evaluation statistics
        """
        acc_all = []
        loss_all = []
        evaluation_stats = []

        n_tasks = len(test_loader)
        for episode_index, (
            support_images,
            support_labels,
            query_images,
            query_labels,
            class_ids,
            source_domain,
            target_domain,
        ) in enumerate(test_loader):

            query_labels = set_device(query_labels)
            support_labels = set_device(support_labels)

            if self.use_test_query and \
              (self.adv_train_config is not None):
                loss_fn = self.transductive_loss_fn
                support_images, support_labels, query_images, query_labels, adv_tasks = self.adv_project(
                    'test', episode_index, support_images, support_labels, query_images, query_labels, loss_fn
                )

            scores = self.set_forward(
                support_images, support_labels, query_images
            ).detach()

            loss_value = self.loss_fn(scores, query_labels).detach().item()

            evaluation_stats.append(
                self.get_task_perf(
                    episode_index,
                    scores.cpu(),
                    query_labels.cpu().detach(),
                    class_ids,
                    source_domain,
                    target_domain,
                )
            )

            loss_all.append(loss_value)

            acc_all.append(self.evaluate(scores, query_labels) * 100)

        evaluations_stats_df = pd.concat(evaluation_stats, ignore_index=True)
        acc_all = np.asarray(acc_all)
        acc_mean = np.mean(acc_all)
        acc_std = np.std(acc_all)
        logger.info(
            "%d Test Accuracy = %4.2f%% +- %4.2f%%"
            % (n_tasks, acc_mean, confidence_interval(acc_std, n_tasks))
        )

        return np.asarray(loss_all), acc_all, evaluations_stats_df