from __future__ import print_function

import os
import sys

import pickle
import argparse
import numpy as np

import torch
import torch.nn as nn
from torch.utils import data
from torch.autograd import Variable
import torch.optim as optim
import torch.nn as nn
import torch.nn.functional as F
import torch.backends.cudnn as cudnn

from tensorboardX import SummaryWriter

# from utils.loss import CrossEntropy2d, BCEWithLogitsLoss2d

from dataset.shapeNetData import ShapeNetDataset, ShapeNetGTDataset

from models.pointnet import PointNetSeg
from models.discriminator import ConvDiscNet

import timeit
start = timeit.default_timer()

BASE_DIR = os.path.dirname(os.path.abspath(__file__))

MODEL = 'PointNetSeg'
BATCH_SIZE = 16
ITER_SIZE = 1
NUM_WORKERS = 6

LEARNING_RATE = 0.0001
MOMENTUM = 0.9
NUM_INST_CLASSES = 16
NUM_SEG_CLASSES = 50
NUM_STEPS = 175088 # 20000
POWER = 0.9
# RANDOM_SEED = 1234

SAVE_PRED_EVERY = 1750 # every 2 epoches, batch size 16
SNAPSHOT_DIR = './snapshots/'
WEIGHT_DECAY = 0.0005

LEARNING_RATE_D = 1e-4
LAMBDA_ADV_PRED = 0.01

PARTIAL_DATA=0.25

SEMI_START=8754 #10 epochs
LAMBDA_SEMI=0.1
MASK_T=0.1

LAMBDA_SEMI_ADV=0.001
SEMI_START_ADV=0 # 0
D_REMAIN=True

parser = argparse.ArgumentParser(description="PointNet-Semi Network")
parser.add_argument("--model", type=str, default=MODEL,
                    help="available options : PointNet")
parser.add_argument("--batch-size", type=int, default=BATCH_SIZE,
                    help="Number of images sent to the network in one step.")
parser.add_argument("--iter-size", type=int, default=ITER_SIZE,
                    help="Accumulate gradients for ITER_SIZE iterations.")
parser.add_argument("--num-workers", type=int, default=NUM_WORKERS,
                    help="number of workers for multithread dataloading.")
parser.add_argument("--partial-data", type=float, default=PARTIAL_DATA,
                    help="The index of the label to ignore during the training.")
parser.add_argument("--partial-id", type=str, default=None,
                    help="restore partial id list")
parser.add_argument("--is-training", action="store_true",
                    help="Whether to updates the running means and variances during the training.")
parser.add_argument("--learning-rate", type=float, default=LEARNING_RATE,
                    help="Base learning rate for training with polynomial decay.")
parser.add_argument("--learning-rate-D", type=float, default=LEARNING_RATE_D,
                    help="Base learning rate for discriminator.")
parser.add_argument("--lambda-adv-pred", type=float, default=LAMBDA_ADV_PRED,
                    help="lambda_adv for adversarial training.")
parser.add_argument("--lambda-semi", type=float, default=LAMBDA_SEMI,
                    help="lambda_semi for adversarial training.")
parser.add_argument("--lambda-semi-adv", type=float, default=LAMBDA_SEMI_ADV,
                    help="lambda_semi for adversarial training.")
parser.add_argument("--mask-T", type=float, default=MASK_T,
                    help="mask T for semi adversarial training.")
parser.add_argument("--semi-start", type=int, default=SEMI_START,
                    help="start semi learning after # iterations")
parser.add_argument("--semi-start-adv", type=int, default=SEMI_START_ADV,
                    help="start semi learning after # iterations")
parser.add_argument("--D-remain", type=bool, default=D_REMAIN,
                    help="Whether to train D with unlabeled data")
parser.add_argument("--momentum", type=float, default=MOMENTUM,
                    help="Momentum component of the optimiser.")
parser.add_argument("--not-restore-last", action="store_true",
                    help="Whether to not restore last (FC) layers.")
parser.add_argument("--num-instance-classes", type=int, default=NUM_INST_CLASSES,
                    help="Number of classes to predict (including background).")
parser.add_argument("--num-seg-classes", type=int, default=NUM_SEG_CLASSES,
                    help="Number of classes to predict (including background).")
parser.add_argument("--num-steps", type=int, default=NUM_STEPS,
                    help="Number of training steps.")
parser.add_argument("--power", type=float, default=POWER,
                    help="Decay parameter to compute the learning rate.")
parser.add_argument("--random-mirror", action="store_true",
                    help="Whether to randomly mirror the inputs during the training.")
parser.add_argument("--random-scale", action="store_true",
                    help="Whether to randomly scale the inputs during the training.")
# parser.add_argument("--random-seed", type=int, default=RANDOM_SEED,
#                     help="Random seed to have reproducible results.")
parser.add_argument("--restore-from-D", type=str, default=None,
                    help="Where restore model parameters from.")
parser.add_argument("--save-pred-every", type=int, default=SAVE_PRED_EVERY,
                    help="Save summaries and checkpoint every often.")
parser.add_argument("--snapshot-dir", type=str, default=SNAPSHOT_DIR,
                    help="Where to save snapshots of the model.")
parser.add_argument("--weight-decay", type=float, default=WEIGHT_DECAY,
                    help="Regularisation parameter for L2-loss.")
parser.add_argument("--gpu", type=int, default=0,
                    help="choose gpu device.")
parser.add_argument("--num-pts", type=int, dest="num_pts", default=2048,
                    help="#points in each instance")
parser.add_argument("--noise", action="store_true", dest="noise", default=False,
                   help="add noise in data augmentation")
parser.add_argument("--rotate", action="store_true", dest="rotate", default=False,
                   help="rotate shape in data augmentation")

opts = parser.parse_args()
print(opts)

results_folder = os.path.join(BASE_DIR, "results")
if not os.path.isdir(results_folder):
    os.mkdir(results_folder)
writer_folder = os.path.join(results_folder, "logs")
if not os.path.isdir(writer_folder):
    os.mkdir(writer_folder)
writer = SummaryWriter(writer_folder)

class CrossEntropyMask(nn.Module):

    def __init__(self, size_average=True, ignore_label=999):
        super(CrossEntropyMask, self).__init__()
        self.size_average = size_average
        self.ignore_label = ignore_label

    def forward(self, predict, target, weight=None):
        """
            Args:
                predict:(b, c, npts)
                target:(b, npts)
                weight (Tensor, optional): a manual rescaling weight given to each class.
                                           If given, has to be a Tensor of size "nclasses"
        """
        assert not target.requires_grad
        assert predict.dim() == 3
        assert target.dim() == 2
        assert predict.size(0) == target.size(0), "{0} vs {1} ".format(predict.size(0), target.size(0))
        assert predict.size(2) == target.size(1), "{0} vs {1} ".format(predict.size(2), target.size(1))

        b, c, npts = predict.size()
        target_mask = (target >= 0) * (target != self.ignore_label)
        target = target[target_mask] # (bxnpts)xc
        if not target.data.dim():
            return Variable(torch.zeros(1))
        predict = predict.transpose(1, 2).contiguous()
        predict = predict[target_mask.view(b, npts, 1).repeat(1, 1, c)].view(-1, c)
        loss = F.cross_entropy(predict, target, weight=weight, size_average=self.size_average)
        return loss

def loss_calc(pred, label, device, mask=False):
    """
    This function returns cross entropy loss for semantic segmentation
    """
    # out shape batch_size x channels x h x w -> batch_size x channels x h x w
    # label shape h x w x 1 x batch_size  -> batch_size x 1 x h x w
    pred = pred.transpose(2,1)
    label = Variable(label.long()).to(device)
    if not mask:
        criterion = nn.CrossEntropyLoss().to(device)
    else:
        criterion = CrossEntropyMask().to(device)

    return criterion(pred, label)

def loss_bce(pred, label, device):
    """
    :return: BCE loss
    """
    label = Variable(label.float()).to(device)
    criterion = nn.BCEWithLogitsLoss().to(device)

    return criterion(pred, label)

def lr_poly(base_lr, iter, max_iter, power):
    return base_lr*((1-float(iter)/max_iter)**(power))

def adjust_learning_rate(optimizer, i_iter):
    lr = lr_poly(opts.learning_rate, i_iter, opts.num_steps, opts.power)
    optimizer.param_groups[0]['lr'] = lr
    if len(optimizer.param_groups) > 1 :
        optimizer.param_groups[1]['lr'] = lr * 10

def adjust_learning_rate_D(optimizer, i_iter):
    lr = lr_poly(opts.learning_rate_D, i_iter, opts.num_steps, opts.power)
    optimizer.param_groups[0]['lr'] = lr
    if len(optimizer.param_groups) > 1 :
        optimizer.param_groups[1]['lr'] = lr * 10

def one_hot(label, num_classes):
    label = label.numpy()
    one_hot = np.zeros((label.shape[0], label.shape[1], num_classes), dtype=label.dtype)
    # for i in range(num_classes):
    #     one_hot[:,i,...] = (label==i)
    for b in range(label.shape[0]):
        for p in range(label.shape[1]):
            one_hot[b, p, label[b,p]] = 1
    #handle ignore labels
    # return torch.FloatTensor(one_hot)
    return torch.tensor(one_hot, dtype=torch.float)

def make_D_label(label, label_shape, device):
    # ignore_mask = np.expand_dims(label_shape, axis=1)
    D_label = np.ones(label_shape.shape)*label
    # D_label[ignore_mask] = 255
    # D_label = Variable(torch.FloatTensor(D_label)).cuda(args.gpu)
    D_label = Variable(torch.tensor(D_label, dtype=torch.float)).to(device)

    return D_label

def main():
    cudnn.enabled = True
    # create network
    model = PointNetSeg(NUM_SEG_CLASSES)

    writer.add_graph(model, (Variable(torch.Tensor(1,2048,3), requires_grad=True),
                              Variable(torch.Tensor(1,1,16), requires_grad=True)))

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    model.to(device)
    model.train()

    cudnn.benchmark = True

    # init D
    model_D = ConvDiscNet(input_dim=NUM_SEG_CLASSES)
    writer.add_graph(model_D, Variable(torch.Tensor(1, 2048, 50), requires_grad=True))
    writer.close()

    if opts.restore_from_D is not None:
        model_D.load_state_dict(torch.load(opts.restore_from_D))
    model_D.to(device)
    model_D.train()

    if not os.path.exists(opts.snapshot_dir):
        os.makedirs(opts.snapshot_dir)

    train_dataset = ShapeNetDataset(num_classes=NUM_INST_CLASSES,
                                    npts=opts.num_pts,
                                    mode = "train",
                                    aug_noise = opts.noise,
                                    aug_rotate = opts.rotate)
    train_dataset_size = len(train_dataset)
    print("#Total train: {:6d}".format(train_dataset_size))

    train_gt_dataset = ShapeNetGTDataset(num_classes=NUM_INST_CLASSES,
                                         npts=opts.num_pts)

    if opts.partial_data is None:
        trainloader = data.DataLoader(train_dataset,
                        batch_size=opts.batch_size, shuffle=True, num_workers=NUM_WORKERS, pin_memory=True)

        trainloader_gt = data.DataLoader(train_gt_dataset,
                        batch_size=opts.batch_size, shuffle=True, num_workers=NUM_WORKERS, pin_memory=True)
    else:
        partial_size = int(opts.partial_data * train_dataset_size)

        if opts.partial_id is not None:
            train_ids = pickle.load(open(opts.partial_id))
            print('loading train ids from {}'.format(opts.partial_id))
        else:
            train_ids = list(range(train_dataset_size))
            np.random.shuffle(train_ids)

        pickle.dump(train_ids, open(os.path.join(opts.snapshot_dir, 'train_id.pkl'), 'wb'))

        train_sampler = data.sampler.SubsetRandomSampler(train_ids[:partial_size])
        train_remain_sampler = data.sampler.SubsetRandomSampler(train_ids[partial_size:])
        train_gt_sampler = data.sampler.SubsetRandomSampler(train_ids[:partial_size])

        trainloader = data.DataLoader(train_dataset,
                        batch_size=opts.batch_size, sampler=train_sampler, num_workers=NUM_WORKERS, pin_memory=True)
        trainloader_remain = data.DataLoader(train_dataset,
                        batch_size=opts.batch_size, sampler=train_remain_sampler, num_workers=NUM_WORKERS, pin_memory=True)
        trainloader_gt = data.DataLoader(train_gt_dataset,
                        batch_size=opts.batch_size, sampler=train_gt_sampler, num_workers=NUM_WORKERS, pin_memory=True)

        trainloader_remain_iter = iter(trainloader_remain)

    trainloader_iter = iter(trainloader)
    trainloader_gt_iter = iter(trainloader_gt)

    # implement model.optim_parameters(args) to handle different models' lr setting

    # optimizer for segmentation network
    optimizer = optim.Adam(model.parameters(), lr=0.001)
    optimizer.zero_grad()

    # optimizer for discriminator network
    optimizer_D = optim.Adam(model_D.parameters(), lr=0.001, betas=(0.9,0.999))
    optimizer_D.zero_grad()

    # labels for adversarial training
    pred_label = 0
    gt_label = 1

    for i_iter in range(opts.num_steps):

        loss_seg_value = 0
        loss_adv_pred_value = 0
        loss_D_value = 0
        loss_semi_value = 0
        loss_semi_adv_value = 0

        optimizer.zero_grad()
        adjust_learning_rate(optimizer, i_iter)
        optimizer_D.zero_grad()
        adjust_learning_rate_D(optimizer_D, i_iter)

        for sub_i in range(opts.iter_size):

            # train G

            # don't accumulate grads in D
            for param in model_D.parameters():
                param.requires_grad = False

            # do semi first
            if (opts.lambda_semi > 0 or opts.lambda_semi_adv > 0 ) and i_iter >= opts.semi_start_adv :
                try:
                    batch = next(trainloader_remain_iter)
                except:
                    trainloader_remain_iter = iter(trainloader_remain)
                    batch = next(trainloader_remain_iter)

                # only access to img
                points, cls, _ = batch
                points, cls = Variable(points).float(), Variable(cls).float()
                points, cls = points.to(device), cls.to(device)

                pred = model(points, cls) # BxNxC
                pred_remain = pred.detach()

                D_out = model_D(F.softmax(pred, dim=2))
                D_out_sigmoid = torch.sigmoid(D_out).data.cpu().numpy() # BxN
                ignore_mask_remain = np.zeros(D_out_sigmoid.shape).astype(np.bool) # Bx2048

                loss_semi_adv = opts.lambda_semi_adv * loss_bce(D_out, make_D_label(gt_label, ignore_mask_remain, device), device)
                loss_semi_adv = loss_semi_adv/opts.iter_size

                # writer.add_scalar('loss_semi_adv', loss_semi_adv.item(), i_iter)

                loss_semi_adv_value += loss_semi_adv.item() / opts.lambda_semi_adv

                if opts.lambda_semi <= 0 or i_iter < opts.semi_start:
                    loss_semi_adv.backward()
                    loss_semi_value = 0
                else:
                    # produce ignore mask
                    semi_ignore_mask = (D_out_sigmoid < opts.mask_T)

                    semi_gt = pred.data.cpu().numpy().argmax(axis=2)
                    semi_gt[semi_ignore_mask] = 999

                    semi_ratio = 1.0 - float(semi_ignore_mask.sum())/semi_ignore_mask.size
                    print('semi ratio: {:.4f}'.format(semi_ratio))

                    if semi_ratio == 0.0:
                        loss_semi_value += 0
                    else:
                        semi_gt = torch.FloatTensor(semi_gt)

                        loss_semi = opts.lambda_semi * loss_calc(pred, semi_gt, device, mask=True)
                        loss_semi = loss_semi/opts.iter_size
                        # writer.add_scalar('loss_semi', loss_semi.item(), i_iter)
                        loss_semi_value += loss_semi.item() / opts.lambda_semi
                        loss_semi += loss_semi_adv
                        loss_semi.backward()

            else:
                loss_semi = None
                loss_semi_adv = None

            # train with source

            try:
                # _, batch = trainloader_iter.next()
                batch = next(trainloader_iter)
            except:
                trainloader_iter = iter(trainloader)
                batch = next(trainloader_iter)

            points, cls_gt, seg_gt = batch
            points, cls_gt, seg_gt = Variable(points).float(), Variable(cls_gt).float(), Variable(seg_gt).type(torch.LongTensor)
            points, cls_gt, seg_gt = points.to(device), cls_gt.to(device), seg_gt.to(device)
            # ignore_mask = (seg_gt.numpy() == 255)
            ignore_mask = np.zeros(seg_gt.shape).astype(np.bool)
            pred = model(points, cls_gt) # [5,21,41,41] -> [5, 21, 321, 321]

            loss_seg = loss_calc(pred, seg_gt, device, mask=False)
            # writer.add_scalar('loss_seg', loss_seg.item(), i_iter)

            D_out = model_D(F.softmax(pred, dim=2)) # [5, 1, 10, 10] -> [5, 1, 321, 321]

            loss_adv_pred = loss_bce(D_out, make_D_label(gt_label, ignore_mask, device), device)
            # writer.add_scalar('loss_adv_pred', loss_adv_pred.item(), i_iter)

            loss = loss_seg + opts.lambda_adv_pred * loss_adv_pred

            # proper normalization
            loss = loss/opts.iter_size
            loss.backward()
            loss_seg_value += loss_seg.item() / opts.iter_size
            loss_adv_pred_value += loss_adv_pred.item() / opts.iter_size

            # train D

            # bring back requires_grad
            for param in model_D.parameters():
                param.requires_grad = True

            # train with pred
            pred = pred.detach()

            if opts.D_remain:
                try:
                    pred_remain
                    pred = torch.cat((pred, pred_remain), 0)
                    ignore_mask = np.concatenate((ignore_mask, ignore_mask_remain), axis=0)
                except NameError:
                    print("No pred_remain during training D")

            D_out = model_D(F.softmax(pred, dim=2))
            loss_D = loss_bce(D_out, make_D_label(pred_label, ignore_mask, device), device)
            # loss_D = bce_loss(D_out, make_D_label(pred_label, ignore_mask))
            loss_D = loss_D/opts.iter_size/2
            loss_D_pred = loss_D.item()
            # writer.add_scalar('loss_D_pred', loss_D_pred, i_iter)
            loss_D.backward()
            loss_D_value += loss_D.item()

            # train with gt
            # get gt labels
            try:
                batch = next(trainloader_gt_iter)
            except:
                trainloader_gt_iter = iter(trainloader_gt)
                batch = next(trainloader_gt_iter)

            _, cls_gt, seg_gt = batch
            D_gt_v = Variable(one_hot(seg_gt, NUM_SEG_CLASSES)).float().to(device)
            ignore_mask_gt = np.zeros(seg_gt.shape).astype(np.bool)
            # ignore_mask_gt = (seg_gt.numpy() == 255)

            D_out = model_D(D_gt_v)
            loss_D = loss_bce(D_out, make_D_label(gt_label, ignore_mask_gt, device), device)
            loss_D = loss_D/opts.iter_size/2
            # writer.add_scalar('loss_D_gt', loss_D.item(), i_iter)
            # writer.add_scalar('loss_D_total', loss_D.item()+loss_D_pred, i_iter)
            loss_D.backward()
            loss_D_value += loss_D.item()

        optimizer.step()
        optimizer_D.step()

        print('exp = {}'.format(opts.snapshot_dir))
        print('iter = {0:8d}/{1:8d}, \
                loss_seg = {2:.3f}, \
                loss_adv_p = {3:.3f}, \
                loss_D = {4:.3f}, \
                loss_semi = {5:.3f}, \
                loss_semi_adv = {6:.3f}'.format(
                    i_iter, opts.num_steps,
                    loss_seg_value,
                    loss_adv_pred_value,
                    loss_D_value,
                    loss_semi_value,
                    loss_semi_adv_value))

        if i_iter >= opts.num_steps-1:
            print('save model ...')
            torch.save(model.state_dict(),os.path.join(opts.snapshot_dir, 'ShapeNet_'+str(opts.num_steps)+'.pth'))
            torch.save(model_D.state_dict(),os.path.join(opts.snapshot_dir, 'ShapeNet_'+str(opts.num_steps)+'_D.pth'))
            break

        if i_iter % opts.save_pred_every == 0 and i_iter!=0:
            print('taking snapshot ...')
            torch.save(model.state_dict(),os.path.join(opts.snapshot_dir, 'ShapeNet_'+str(i_iter)+'.pth'))
            torch.save(model_D.state_dict(),os.path.join(opts.snapshot_dir, 'ShapeNet_'+str(i_iter)+'_D.pth'))

    end = timeit.default_timer()
    print(end-start,'seconds')
    # writer.close()

if __name__ == '__main__':
    main()


import argparse
import glob
import json
import argparse
import re
import shutil
from pathlib import Path

import mypath
from data_io import pathutils

def get_config_dir(project_name):
    assert isinstance(project_name, str)
    dir_project = Path(mypath.dir_config, project_name)
    pathutils.check_is_dir(dir_project)
    return dir_project

def load_config_args_train(project_name):
    assert isinstance(project_name, str)
    file_config = Path(get_config_dir(project_name), "config_train.json")
    pathutils.check_is_file(file_config)
    args_o = read_args_from_json(file_config)
    return args_o

def load_config_args_predict(project_name):
    assert isinstance(project_name, str)
    file_config = Path(get_config_dir(project_name), "config_pred.json")
    pathutils.check_is_file(file_config)
    args_o = read_args_from_json(file_config)
    return args_o

def read_args_from_json(path):
    pathutils.check_is_file(path)
    with open(path, "r") as file:
        config = json.load(file)
    args_o = argparse.Namespace(**config)
    return args_o

def get_ckpt_dir(project_name):
    assert isinstance(project_name, str)
    dir_ckpt = Path(get_config_dir(project_name), "ckpts")
    return dir_ckpt

def copy_best_ckpt_2_config(project_name, ckpt_path):
    assert isinstance(project_name, str)
    pathutils.check_is_file(ckpt_path)
    dir_ckpt = get_ckpt_dir(project_name)
    shutil.copy(ckpt_path, dir_ckpt)

def get_pt_from_config(project_name):
    dir_config = get_config_dir(project_name)
    fnames_pt = [Path(x).name for x in dir_config.glob("*.pt")]
    if fnames_pt:
        path_pt = Path(dir_config, fnames_pt[0])
        pathutils.check_is_file(path_pt)
        return path_pt
    else:
        raise FileNotFoundError(f"best ckpt not found: project_name: {project_name}")

def get_best_ckpt_path_from_config(project_name, key_best, b_max=False):
    fname_ckpt = _get_best_cktp_name(project_name, key_best, b_max)
    if fname_ckpt:
        path_ckpt = Path(get_ckpt_dir(project_name), fname_ckpt)
        pathutils.check_is_file(path_ckpt)
        return path_ckpt
    else:
        raise FileNotFoundError(
            f"best ckpt not found: project_name: {project_name}, key_best: {key_best}, b_max: {b_max}"
        )

def _get_best_cktp_name(project_name, key_best, b_max):
    assert isinstance(project_name, str)
    assert isinstance(key_best, str)
    assert isinstance(b_max, bool)
    str_re = f"{key_best}[=]([0-9]*[.][0-9]+)"
    reg = re.compile(str_re)

    fnames_ckpt = _get_all_ckpt_names(project_name)

    if b_max:
        val_best = float("-inf")
    else:
        val_best = float("inf")
    idx_best = None
    for idx_curr, fname in enumerate(fnames_ckpt):
        m = reg.search(fname)
        if m:
            val_curr = float(m.group(1))
            if not b_max and val_curr < val_best:
                val_best = min(val_best, val_curr)
                idx_best = idx_curr
            elif b_max and val_curr > val_best:
                val_best = max(val_best, val_curr)
                idx_best = idx_curr

    if idx_best is None:
        return None
    else:
        return fnames_ckpt[idx_best]

def _get_all_ckpt_names(project_name):
    assert isinstance(project_name, str)
    dir_ckpt = Path(mypath.dir_config, project_name, "ckpts")
    pathutils.check_is_dir(dir_ckpt)
    fnames_ckpt = [Path(x).name for x in dir_ckpt.glob("*.ckpt")]
    return fnames_ckpt


import contextlib
import logging
import os
import shutil
from typing import TYPE_CHECKING, Optional

from snapcraft_legacy import formatting_utils
from snapcraft_legacy.internal import errors, mountinfo, project_loader, steps

logger = logging.getLogger(__name__)

if TYPE_CHECKING:
    from snapcraft_legacy.project import Project

def _clean_part(part_name, step, config, staged_state, primed_state):
    if step.next_step() is None:
        template = "Cleaning {step} step for {part}"
    else:
        template = "Cleaning {step} step (and all subsequent steps) for {part}"

    logger.info(template.format(step=step.name, part=part_name))
    config.parts.clean_part(part_name, staged_state, primed_state, step)

    # If we just cleaned the root of a dependency tree. Return all dirty
    # dependents.
    return get_dirty_reverse_dependencies(part_name, step, config)

def get_dirty_reverse_dependencies(part_name, step, config):
    # If other parts are depending on this step of the part, they're now
    # dirty
    dirty_reverse_dependencies = set()
    reverse_dependencies = config.parts.get_reverse_dependencies(
        part_name, recursive=True
    )
    dirty_step = steps.dirty_step_if_dependency_changes(step)
    for reverse_dependency in reverse_dependencies:
        if not reverse_dependency.should_step_run(dirty_step):
            dirty_reverse_dependencies.add(reverse_dependency)

    return dirty_reverse_dependencies

def _clean_parts(part_names, step, config, staged_state, primed_state):
    if not step:
        step = steps.next_step(None)

    for part_name in part_names:
        dirty_parts = _clean_part(part_name, step, config, staged_state, primed_state)
        dirty_part_names = {p.name for p in dirty_parts}

        parts_not_being_cleaned = dirty_part_names.difference(part_names)
        if parts_not_being_cleaned:
            logger.warning(
                "Cleaned {!r}, which makes the following {} out of date: "
                "{}".format(
                    part_name,
                    formatting_utils.pluralize(
                        parts_not_being_cleaned, "part", "parts"
                    ),
                    formatting_utils.humanize_list(parts_not_being_cleaned, "and"),
                )
            )

def _remove_directory(directory: str) -> None:
    if os.path.isdir(directory):
        shutil.rmtree(directory)

def _remove_directory_if_empty(directory):
    if os.path.isdir(directory) and not os.listdir(directory):
        os.rmdir(directory)

def _cleanup_common_directories(config, project: "Project") -> None:
    max_step = None
    for part in config.all_parts:
        with contextlib.suppress(errors.NoLatestStepError):
            step = part.latest_step()
            if not max_step or step > max_step:
                max_step = step

    next_step = steps.next_step(max_step)
    if next_step:
        _cleanup_common_directories_for_step(next_step, project)

def _cleanup_common_directories_for_step(step, project: "Project", parts=None):
    if not parts:
        parts = []

    being_tried = False
    message: Optional[str] = None
    if step <= steps.PRIME:
        # Remove the priming area. Only remove the actual 'prime' directory if
        # it's NOT being used in 'snap try'. We'll know that if it's
        # bind-mounted somewhere.
        mounts = mountinfo.MountInfo()
        try:
            mounts.for_root(project.prime_dir)
        except errors.RootNotMountedError:
            remove_dir = True
            message = "Cleaning up priming area"
        else:
            remove_dir = False
            message = (
                "Cleaning up priming area, but not removing as it's in "
                "use by 'snap try'"
            )
            being_tried = True
        finally:
            # It does not matter if prime is being tried inside the managed-host
            # or not, we cannot delete prime as it is really a mount from the
            # outside host.
            if project._is_managed_host:
                remove_dir = False
                message = None
        _cleanup_common(
            project.prime_dir, steps.PRIME, message, parts, remove_dir=remove_dir
        )

    if step <= steps.STAGE:
        # Remove the staging area.
        _cleanup_common(
            project.stage_dir, steps.STAGE, "Cleaning up staging area", parts
        )

    if step <= steps.PULL:
        # Remove the parts directory (but leave local plugins alone).
        _cleanup_parts_dir(project.parts_dir, project.local_plugins_dir, parts)

    if not being_tried and not project._is_managed_host:
        _remove_directory_if_empty(project.prime_dir)
    _remove_directory_if_empty(project.stage_dir)
    _remove_directory_if_empty(project.parts_dir)

def _cleanup_common(directory, step, message, parts, *, remove_dir=True):
    if os.path.isdir(directory):
        if message is not None:
            logger.info(message)
        if remove_dir:
            shutil.rmtree(directory)
        else:
            # Don't delete the parent directory, but delete its contents
            for f in os.scandir(directory):
                if f.is_dir(follow_symlinks=False):
                    shutil.rmtree(f.path)
                elif f.is_file(follow_symlinks=False):
                    os.remove(f.path)
    for part in parts:
        part.mark_cleaned(step)

def _cleanup_parts_dir(parts_dir, local_plugins_dir, parts):
    if os.path.exists(parts_dir):
        logger.info("Cleaning up parts directory")
        for subdirectory in os.listdir(parts_dir):
            path = os.path.join(parts_dir, subdirectory)
            if path != local_plugins_dir:
                try:
                    shutil.rmtree(path)
                except NotADirectoryError:
                    os.remove(path)
    for part in parts:
        part.mark_cleaned(steps.BUILD)
        part.mark_cleaned(steps.PULL)

def clean(project: "Project", parts, step=None):
    # step defaults to None because that's how it comes from docopt when it's
    # not set.
    if not step:
        step = steps.PULL

    if not parts and step == steps.PULL:
        _cleanup_common_directories_for_step(step, project)
        return

    config = project_loader.load_config(project)

    if not parts and step <= steps.PRIME:
        # If we've been asked to clean stage or prime without being given
        # specific parts, just blow away those directories instead of
        # doing it per part (it would just be a waste of time).
        _cleanup_common_directories_for_step(step, project, parts=config.all_parts)

        # No need to continue if that's all that was required
        if step >= steps.STAGE:
            return

    if parts:
        config.parts.validate(parts)
    else:
        parts = [part.name for part in config.all_parts]

    staged_state = config.get_project_state(steps.STAGE)
    primed_state = config.get_project_state(steps.PRIME)

    _clean_parts(parts, step, config, staged_state, primed_state)

    _cleanup_common_directories(config, project)


"""Describes the AuxiliaryBaseclass from which other model classes inherit.

The AuxiliaryBaseclass describes class attributes and methods shared by
auxiliary tools like kernel_xPCA and kernel_FGen.
"""
import sys

try:
    import cupy as cp
except:
    pass

from .kernels import KERNEL_NAME_TO_CLASS
from .constants import constants

class AuxiliaryBaseclass():
    """The base class for auxiliary toolkits in xGPR, primarily
    for performing kernel PCA and kernel KMeans.

    Attributes:
        kernel: The kernel object for the posterior predictive mean. The class of
            this object will depend on the kernel specified by the user.
        device (str): One of "gpu", "cpu". The user can update this as desired.
            All predict / tune / fit operations are carried out using the
            current device.
        num_rffs (int): The number of random Fourier features generated
            by the auxiliary tool.
        kernel_specific_params (dict): Contains kernel-specific parameters --
            e.g. 'matern_nu' for the nu for the Matern kernel, or 'conv_width'
            for the conv1d kernel.
        random_seed (int): The random seed to use for all future ops.
        verbose (bool): If True, regular updates are printed during
            hyperparameter tuning and fitting.
        num_threads (int): The number of threads to use for random feature generation
            if running on CPU. If running on GPU, this argument is ignored.
        double_precision_fht (bool): If True, use double precision during FHT for
            generating random features. For most problems, it is not beneficial
            to set this to True -- it merely increases computational expense
            with negligible benefit -- but this option is useful for testing.
            Defaults to False.
    """

    def __init__(self, num_rffs, hyperparams, dataset, kernel_choice="RBF", device = "cpu",
                    kernel_specific_params = constants.DEFAULT_KERNEL_SPEC_PARMS,
                    random_seed = 123, verbose = True,
                    num_threads = 2,
                    double_precision_fht = False):
        """Constructor.

        Args:
            num_rffs (int): The number of random Fourier features
                to use for the auxiliary device.
            hyperparams (np.ndarray): A numpy array containing the kernel-specific
                hyperparameter. If you have fitted an xGPR model, the first two
                hyperparameters are general not kernel specific, so
                my_model.get_hyperparams()[2:] will retrieve the hyperparameters you
                need. For most kernels there is only one kernel-specific hyperparameter.
                For kernels with no kernel-specific hyperparameter (e.g. arc-cosine
                and polynomial kernels), this argument is ignored.
            dataset: A valid dataset object.
            kernel_choice (str): The kernel that the model will use.
                Defaults to 'RBF'. Must be in kernels.kernel_list.
                KERNEL_NAME_TO_CLASS.
            device (str): Determines whether calculations are performed on
                'cpu' or 'gpu'. The initial entry can be changed later
                (i.e. model can be transferred to a different device).
                Defaults to 'cpu'.
            kernel_specific_params (dict): Contains kernel-specific parameters --
                e.g. 'matern_nu' for the nu for the Matern kernel, or 'conv_width'
                for the conv1d kernel.
            random_seed (int): A seed for the random number generator.
            verbose (bool): If True, regular updates are printed
                during fitting and tuning. Defaults to True.
            num_threads (int): The number of threads to use for random feature generation
                if running on CPU. If running on GPU, this argument is ignored.
            double_precision_fht (bool): If True, use double precision during FHT for
                generating random features. For most problems, it is not beneficial
                to set this to True -- it merely increases computational expense
                with negligible benefit -- but this option is useful for testing.
                Defaults to False.
        """
        self.random_seed = random_seed
        self.num_rffs = num_rffs
        self.kernel_spec_parms = kernel_specific_params
        #We should never use a y-intercept for kPCA or clustering.
        self.kernel_spec_parms["intercept"] = False
        self.num_threads = num_threads

        self.verbose = verbose

        self.double_precision_fht = double_precision_fht
        if kernel_choice not in KERNEL_NAME_TO_CLASS:
            raise ValueError("An unrecognized kernel choice was supplied.")
        self.kernel = KERNEL_NAME_TO_CLASS[kernel_choice](dataset.get_xdim(),
                            num_rffs, random_seed, device,
                            self.num_threads, self.double_precision_fht,
                            kernel_spec_parms = self.kernel_spec_parms)
        self.device = device
        full_hparams = self.kernel.get_hyperparams()
        if full_hparams.shape[0] > 2:
            full_hparams[2:] = hyperparams
        full_hparams[1] = 0
        self.kernel.set_hyperparams(full_hparams)

    def pre_prediction_checks(self, input_x):
        """Checks input data to ensure validity.

        Args:
            input_x (np.ndarray): A numpy array containing the input data.
            get_var (bool): Whether a variance calculation is desired.

        Returns:
            x_array: A cupy array (if self.device is gpu) or a reference
                to the unmodified input array otherwise.

        Raises:
            ValueError: If invalid inputs are supplied,
                a detailed ValueError is raised to explain.
        """
        x_array = input_x
        if not self.kernel.validate_new_datapoints(input_x):
            raise ValueError("The input has incorrect dimensionality.")
        if self.device == "gpu":
            mempool = cp.get_default_memory_pool()
            mempool.free_all_blocks()
            x_array = cp.asarray(input_x)

        return x_array

    def transform_data(self, input_x, chunk_size = 2000):
        """Generate the random features for each chunk
        of an input array. This function is a generator
        so it will yield the random features as blocks
        of shape (chunk_size, fitting_rffs).

        Args:
            input_x (np.ndarray): The input data. Should be a 2d numpy
                array (if non-convolution kernel) or 3d (if convolution
                kernel).
            chunk_size (int): The number of datapoints to process at
                a time. Lower values limit memory consumption. Defaults
                to 2000.

        Yields:
            x_trans (array): An array containing the random features
                generated for a chunk of the input. Shape is
                (chunk_size, fitting_rffs).

        Raises:
            ValueError: If the dimensionality or type of the input does
                not match what is expected, or if the model has
                not yet been fitted, a ValueError is raised.
        """
        xdata = self.pre_prediction_checks(input_x)
        for i in range(0, xdata.shape[0], chunk_size):
            cutoff = min(i + chunk_size, xdata.shape[0])
            yield self.kernel.transform_x(xdata[i:cutoff, :])

    ####The remaining functions are all getters / setters.

    @property
    def kernel_spec_parms(self):
        """Property definition for the kernel_spec_parms."""
        return self._kernel_spec_parms

    @kernel_spec_parms.setter
    def kernel_spec_parms(self, value):
        """Setter for kernel_spec_parms."""
        if not isinstance(value, dict):
            raise ValueError("Tried to set kernel_spec_parms to something that "
                    "was not a dict!")
        self._kernel_spec_parms = value

    @property
    def num_rffs(self):
        """Property definition for the num_rffs attribute."""
        return self._num_rffs

    @num_rffs.setter
    def num_rffs(self, value):
        """Setter for the num_rffs attribute."""
        self._num_rffs = value

    @property
    def num_threads(self):
        """Property definition for the num_threads attribute."""
        return self._num_threads

    @num_threads.setter
    def num_threads(self, value):
        """Setter for the num_threads attribute."""
        if value > 24 or value < 1:
            self._num_threads = 2
            raise ValueError("Num threads if supplied must be an integer from 1 to 24.")
        self._num_threads = value

    @property
    def double_precision_fht(self):
        """Property definition for the double_precision_fht attribute."""
        return self._double_precision_fht

    @double_precision_fht.setter
    def double_precision_fht(self, value):
        """Setter for the double_precision_fht attribute."""
        self._double_precision_fht = value

    @property
    def device(self):
        """Property definition for the device attribute."""
        return self._device

    @device.setter
    def device(self, value):
        """Setter for the device attribute."""
        if value not in ["cpu", "gpu"]:
            raise ValueError("Device must be in ['cpu', 'gpu'].")

        if "cupy" not in sys.modules and value == "gpu":
            raise ValueError("You have specified the gpu fit mode but CuPy is "
                "not installed. Currently CPU only fitting is available.")

        if "cuda_rf_gen_module" not in sys.modules and value == "gpu":
            raise ValueError("You have specified the gpu fit mode but the "
                "cudaHadamardTransform module is not installed / "
                "does not appear to have installed correctly. "
                "Currently CPU only fitting is available.")

        self.kernel.device = value
        if value == "gpu":
            mempool = cp.get_default_memory_pool()
            mempool.free_all_blocks()
        self._device = value


import numpy as np
from scipy.spatial.distance import pdist, squareform

elements_map = {'H': 0,
                'C': 1,
                'N': 2,
                'O': 3,
                'S': 4,
                'X': 5}

def one_hot_elements_map(x):
    arr = np.zeros(len(elements_map))
    arr[elements_map[x[0]]] = 1
    return arr

def PDBdf2npArray(atoms, res, size=np.Inf):
    num_channels = len(elements_map)

    x_coord = (((atoms.x_coord - atoms.x_coord.min()) / res).astype(int)).values
    y_coord = (((atoms.y_coord - atoms.y_coord.min()) / res).astype(int)).values
    z_coord = (((atoms.z_coord - atoms.z_coord.min()) / res).astype(int)).values
    channel_coord = np.vectorize(lambda x: elements_map[x])(atoms.element_symbol.values)

    max_size = max([x_coord.max(), y_coord.max(), z_coord.max()]) + 1

    if size >= max_size:
        max_size = size
        clip_flag = False
    else:
        clip_flag = True

    shape = (max_size, max_size, max_size, num_channels)

    arr = np.zeros(shape=shape, dtype=np.bool)
    arr[x_coord, y_coord, z_coord, channel_coord] = 1

    if clip_flag:
        arr = arr[:size, :size, :size, :]

    return arr, clip_flag

def PDBdf2linearArray(atoms, size=np.Inf):
    x_coord = (atoms.x_coord - atoms.x_coord.min()).values
    y_coord = (atoms.y_coord - atoms.y_coord.min()).values
    z_coord = (atoms.z_coord - atoms.z_coord.min()).values
    coords = np.stack([x_coord, y_coord, z_coord], axis=-1)

    atom_type = np.apply_along_axis(one_hot_elements_map, -1, np.expand_dims(atoms.element_symbol.values, axis=-1))
    # np.vectorize(one_hot_elements_map)(atoms.element_symbol.values)

    distances = squareform(pdist(coords))
    distances.partition(distances.shape[-1] - 4, axis=-1)
    dist = distances[:, -3:]

    arr = np.concatenate((coords, atom_type, dist), axis=-1)

    return arr

def rotatePDBdf(df, angles):
    # 3D rotation of the x,y,z coordinates in a dataframe

    cos = np.cos(angles)
    sin = np.sin(angles)

    # Rotation matrix
    R = np.array(
            [[cos[1] * cos[2], cos[2] * sin[0] * sin[1] - cos[0] * sin[2], cos[0] * cos[2] * sin[1] + sin[0] * sin[2]],
             [cos[1] * sin[2], cos[0] * cos[2] + sin[0] * sin[1] * sin[2], -cos[2] * sin[0] + cos[0] * sin[1] * sin[2]],
             [-sin[1], cos[1] * sin[0], cos[0] * cos[1]]])

    M = np.array([df.x_coord.values, df.y_coord.values, df.z_coord.values])

    rotM = np.matmul(R, M)

    df.x_coord = rotM[0]
    df.y_coord = rotM[1]
    df.z_coord = rotM[2]

    return df

def rotateArray(arr, angles):
    # 3D rotation of the x,y,z coordinates in a dataframe

    cos = np.cos(angles)
    sin = np.sin(angles)

    # Rotation matrix
    R = np.array(
            [[cos[1] * cos[2], cos[2] * sin[0] * sin[1] - cos[0] * sin[2], cos[0] * cos[2] * sin[1] + sin[0] * sin[2]],
             [cos[1] * sin[2], cos[0] * cos[2] + sin[0] * sin[1] * sin[2], -cos[2] * sin[0] + cos[0] * sin[1] * sin[2]],
             [-sin[1], cos[1] * sin[0], cos[0] * cos[1]]])

    M = arr[:, :3]

    rotM = np.matmul(R, M.transpose())

    rot_arr = np.concatenate((rotM.transpose(), arr[:, 3:]), axis=-1)

    return rot_arr


#!/usr/bin/env python

import logging
import gc
from mem import memory_usage, all_info

logging.basicConfig(
    format='%(asctime)-15s  %(levelname)s  %(filename)s:%(lineno)d  %(message)s',
    level=logging.DEBUG,
)

def make_big_list():
    x = [1.0]
    for i in range(6):
        y = []
        for _ in xrange(10):
            y.extend(x)
        x = y
    return x

# http://stackoverflow.com/questions/23369937

"""
Long running Python jobs that consume a lot of memory while
running may not return that memory to the operating system
until the process actually terminates, even if everything
is garbage collected properly.

The solution I need for my problem is that FOR EACH PIECE,
I'll need to spawn a new process AND TERMINATE THAT PROCESS
when its work is done. I think that's the only way out of this
morass.
"""

def main():
    for _ in xrange(5):
        logging.debug(memory_usage())
        logging.debug(all_info(gc.get_objects()))
        x = make_big_list()
        logging.debug(memory_usage())
        logging.debug(all_info(gc.get_objects()))
        del x
        logging.debug(memory_usage())
        logging.debug(all_info(gc.get_objects()))

if __name__ == "__main__":
    main()


from django.urls import path
from .views import ItemCreateView,ItemCategoryListView,ItemCategoryEditView,CategoryRemoveView,ProductCreateView,ProductListView,ProductEditView,ProductRemoveView,ViewCustomerOrders,LogInView,LogoutView,StatusUpdateView

urlpatterns = [
    path('additem',ItemCreateView.as_view(),name="additem"),
    path('items',ItemCategoryListView.as_view(),name="listitems"),
    path('items/<int:pk>',ItemCategoryEditView.as_view(),name="updatecategory"),
    path("itemremove/<int:pk>",CategoryRemoveView.as_view(),name="removecategory"),
    path("addproduct",ProductCreateView.as_view(),name="addproduct"),
    path("products",ProductListView.as_view(),name="products"),
    path("product/<int:pk>",ProductEditView.as_view(),name="updateproduct"),
    path("productremove/<int:pk>",ProductRemoveView.as_view(),name="removeproduct"),
    path("orders",ViewCustomerOrders.as_view(),name="orders"),
    path("login",LogInView.as_view(),name="login"),
    path("logout",LogoutView.as_view(),name="logout"),
    path("status/<int:pk>",StatusUpdateView.as_view(),name="status")
    ]


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Sep 27 14:15:41 2018

@author: Zach
"""

from __future__ import print_function
import os

from apiclient import discovery
from httplib2 import Http
from oauth2client import file, client, tools
import xml.etree.ElementTree as ET
import tempfile
import xmltodict
import json

import numpy as np
import requests

def indent(elem, level=0):
  i = "\n" + level*"  "
  if len(elem):
    if not elem.text or not elem.text.strip():
      elem.text = i + "  "
    if not elem.tail or not elem.tail.strip():
      elem.tail = i
    for elem in elem:
      indent(elem, level+1)
    if not elem.tail or not elem.tail.strip():
      elem.tail = i
  else:
    if level and (not elem.tail or not elem.tail.strip()):
      elem.tail = i

def downloader(FILENAME):
    SCOPES = 'https://www.googleapis.com/auth/drive.readonly'
    store = file.Storage('storage.json')
    creds = store.get()
    path = os.path.dirname(__file__)

    if not creds or creds.invalid:
        secret = os.path.join(path,'client_secret.json')
        flow = client.flow_from_clientsecrets(secret, SCOPES)
        creds = tools.run_flow(flow, store)
    DRIVE = discovery.build('drive', 'v3', http=creds.authorize(Http()))

    SRC_MIMETYPE = 'application/vnd.google-apps.spreadsheet'
    DST_MIMETYPE = 'text/tab-separated-values'
    with tempfile.TemporaryDirectory() as tsvContainer:
        files = DRIVE.files().list(
            q='name="%s" and mimeType="%s"' % (FILENAME, SRC_MIMETYPE),
            orderBy='modifiedTime desc,name').execute().get('files', [])

        if files:
            fn = os.path.join(tsvContainer,'NautilusMat.tsv')
            print('Exporting "%s" as "%s"... ' % (files[0]['name'], fn), end='')
            data = DRIVE.files().export(fileId=files[0]['id'], mimeType=DST_MIMETYPE).execute()
            if data:
                with open(fn, 'wb') as f:
                    f.write(data)
                print('DONE')
            else:
                print('ERROR (could not download file)')
        else:
            print('!!! ERROR: File not found')

        dirName = os.path.join(path,'materials')
        if not os.path.exists(dirName):
            os.mkdir(dirName)
            print("Directory " , dirName ,  " Created ")
        else:
            print("Directory " , dirName ,  " already exists")

        sheet = open(fn)
        sheetData = np.genfromtxt(sheet, delimiter = '\t', dtype=np.dtype(('U', 512)),filling_values=1)
        enabledData = sheetData[1,1:]
        excTitleData = []
        for i in range(1,len(sheetData[1])):
            excTitleData.append(('hr_'+sheetData[2,i]+'_'+sheetData[3,i]+'.xml.fdm_material').lower())
        #excData = np.vstack((np.array(excTitleData),sheetData[53,1:],sheetData[75,1:],sheetData[84,1:])) #make sure these are the lines where the hardware compatible for 0.4mm nozzle is for each printer
        skipPoint = np.where(enabledData == "no")[0]
        sheetData = np.delete(sheetData, (0,1), axis=0)
        catTitles = sheetData[:,0]
        matcostdata = '{'
        #exclusionCrafter(excData)
        for i in range(1,len(sheetData[1])):
            if i-1 in skipPoint:
                #print(i)
                continue
            else:
                catData = [s.replace('@',' ') for s in sheetData[:,i]]
                GUIDdata = catData[5]
                Costdata = catData[13]
                Weightdata = catData[14]
                matcostdata+='\"'+GUIDdata+'\": {\"spool_weight\": '+Weightdata+', \"spool_cost\": '+Costdata+'},'
                xmlCrafter(catTitles,catData,dirName)
        matcostdata = matcostdata[:-1]+'}'
        myfile = open(os.path.join(path,'matCosts.txt'),'w')
        myfile.write(matcostdata)

    return

def xmlCrafter(titles,data,directory):
    fdmmaterial = ET.Element('fdmmaterial')
    metadata = ET.SubElement(fdmmaterial, 'metadata')
    name = ET.SubElement(metadata, 'name')
    properties = ET.SubElement(fdmmaterial, 'properties')
    settings = ET.SubElement(fdmmaterial,'settings')
    machine = ET.SubElement(settings,'machine')
    machine2 = ET.SubElement(settings,'machine')
    headers = [name, metadata, properties, settings, machine, machine2]
    i=k=0
    colonSignal = list(titles).index("material_flow")

    for j in range(len(titles)):
        print(titles[j]+' for k ='+str(k))
        if titles[j] == '':
            k+=1
            i = 0
            if k==1 or k == 3:
                i+=1
            continue
        if titles[j][0]=='-':
            print('hyphen')
            if 'nautilus' in titles[j]:
                machine_identifier = ET.SubElement(machine, 'machine_identifier', manufacturer='Hydra Research', product='Hydra Research Nautilus')
                mac = machine
            elif 'm2' in titles[j]:
                machine_identifier = ET.SubElement(machine, 'machine_identifier', manufacturer='Hydra Research', product='Hydra Research Minnow M2')
                mac = machine
            elif 'minnow' in titles[j]:
                machine_identifier1 = ET.SubElement(machine2, 'machine_identifier', manufacturer='Hydra Research', product='Hydra Research Minnow')
                mac = machine2
        if data[j] == '':
            continue
        elif k < 3:
            headers[k].append(ET.Element(titles[j]))
            headers[k][i].text = data[j]
            i += 1
        elif k == 3:
            if j < colonSignal:
                headers[k].append(ET.Element('setting', key = titles[j]))
            else:
                headers[k].append(ET.Element('cura:setting', key = titles[j]))
            headers[k][i+1].text = data[j]
            #print('setting: '+str(titles[j])+' to: '+str(data[j]))
            #print(k)
            i+=1
        elif k > 3:
            if 'hot' in titles[j]:
                l = 0
                hotend = ET.SubElement(mac, titles[j], id = data[j])
            elif '_' in titles[j]:
                hotend.append(ET.Element('cura:setting', key = titles[j]))
                hotend[l].text = data[j]
                l+=1
            else:
                hotend.append(ET.Element('setting', key = titles[j]))
                hotend[l].text = data[j]
                l+=1

    titlename = os.path.join(directory,'hr_'+data[0]+'_'+data[3]+'.xml.fdm_material').replace(' ','_').lower().replace('-','')
    indent(fdmmaterial)
    mydata = ET.tostring(fdmmaterial,encoding="unicode",method='xml')
    mydata = mydata.replace('<fdmmaterial>', '<?xml version="1.0" encoding="UTF-8"?>\n<fdmmaterial xmlns="http://www.ultimaker.com/material" version="1.3" xmlns:cura="http://www.ultimaker.com/cura">').replace('<color_code>','<color_code>#')
    myfile = open(titlename,'w')
    myfile.write(mydata)
    return

def exclusionCrafter(data):
    nautilusExclusion = []
    minnowExclusion = []
    minnowM2Exclusion = []
    path = os.path.dirname(__file__)

    i=0
    j=0
    k=0
    while i < len(data[0,:]):
        if 'no' in data[1,i]:
            nautilusExclusion.append(data[0,i])
        i += 1

    while j < len(data[0,:]):
        if 'no' in data[2,j]:
            minnowExclusion.append(data[0,j])
        j += 1

    while k < len(data[0,:]):
        if 'no' in data[3,k]:
            minnowM2Exclusion.append(data[0,k])
        k += 1

    folder = os.path.join(path,'exclusion')
    if not os.path.exists(folder):
        os.mkdir(folder)
        print("Directory " , folder ,  " Created ")
    else:
        print("Directory " , folder ,  " already exists")

    with open('hydra_research_excluded_materials.json','r') as f:
        doc = f.read()
        obj = json.loads(doc)

        obj['metadata']['exclude_materials'] = str(nautilusExclusion)

        with open(os.path.join(folder,'hydra_research_nautilus_excluded_materials.def.json'),'w') as g:
            g.write(json.dumps(obj,indent=4))
            g.close()

        obj['metadata']['exclude_materials'] = str(minnowExclusion)

        with open(os.path.join(folder,'hydra_research_minnow_excluded_materials.def.json'),'w') as g:
            g.write(json.dumps(obj,indent=4))
            g.close()

        obj['metadata']['exclude_materials'] = str(minnowM2Exclusion)

        with open(os.path.join(folder,'hydra_research_minnow_m2_excluded_materials.def.json'),'w') as g:
            g.write(json.dumps(obj,indent=4))
            g.close()
        f.close()

downloader('Hydra Research Materials')
#downloader('Test Material')


import logging

from openupgradelib import openupgrade

import odoo

_logger = logging.getLogger(__name__)

def load_modules(registry, force_demo=False, status=None, update_module=False):
    with registry.cursor() as cr:
        # create a technical table
        # to track number of rows per table before and after the migration
        if not openupgrade.table_exists(cr, "openupgrade_track_row_count"):
            cr.execute(
                """
                CREATE TABLE IF NOT EXISTS openupgrade_track_row_count (
                    id serial NOT NULL,
                    table_name character varying(128) NOT NULL,
                    old_row_count integer DEFAULT 0,
                    new_row_count integer
                );
                """,
            )
            # update old rows count of all table before running the migration
            cr.execute(
                """
                WITH tbl AS (
                  SELECT table_name
                  FROM information_schema.tables
                  WHERE table_name not like 'pg_%'
                      AND table_name not like 'ir_%'
                      AND table_name not like 'test_%'
                      AND table_name not like '%_test'
                      AND table_name != 'openupgrade_track_row_count'
                      AND table_schema = 'public'
                )
                INSERT INTO openupgrade_track_row_count (table_name, old_row_count)
                SELECT
                  table_name,
                  (xpath('/row/c/text()',
                    query_to_xml(format('select count(*) AS c from %I', table_name),
                    false,
                    true,
                    '')))[1]::text::int AS old_row_count
                FROM tbl;
                """,
            )

    load_modules._original_method(
        registry,
        force_demo=force_demo,
        status=status,
        update_module=update_module,
    )

    with registry.cursor() as cr:
        # update new number of rows of all table after running the migration
        cr.execute(
            """
            WITH tbl AS (
              SELECT id, table_name
              FROM openupgrade_track_row_count
            )
            UPDATE openupgrade_track_row_count otrc
            SET new_row_count =
              (xpath('/row/c/text()',
                query_to_xml(format('select count(*) AS c from %I', otrc.table_name),
                false,
                true,
                '')))[1]::text::int
            FROM tbl
            WHERE (
                SELECT 1
                FROM information_schema.tables
                WHERE table_name=otrc.table_name
                ) > 0
                AND otrc.id = tbl.id;
            """,
        )
    with registry.cursor() as cr:
        cr.execute(
            """
            SELECT table_name, old_row_count, new_row_count
            FROM openupgrade_track_row_count
            WHERE old_row_count != new_row_count
            """,
        )
        result = cr.dictfetchall()
        if bool(result):

            def msg(col1, col2, col3):
                return "%s%s| %s%s| %s%s\n" % (
                    col1,
                    " " * (50 - len(col1)),
                    col2,
                    " " * (15 - len(str(col2))),
                    col3,
                    " " * (15 - len(str(col3))),
                )

            msg_all = msg("Table name", "Old row count", "New row count")
            for r in result:
                msg_all += msg(r["table_name"], r["old_row_count"], r["new_row_count"])
            _logger.warning(
                "The following tables have different number of rows:\n" "%s\n" % msg_all
            )

load_modules._original_method = odoo.modules.load_modules
odoo.modules.load_modules = load_modules


import csv
import datetime

menuu = "Please Choose a Pizza Base:  " \
        "\n1: Classic" \
        "\n2: Margherita" \
        "\n3: TurkPizza(lahmacun)" \
        "\n4: Pepperoni" \
        "\n5: Four Seasons Pizza" \
        "\n And sauce of your choice:  " \
        "\n1: Olives" \
        "\n2: Mushrooms" \
        "\n3: GoatCheese" \
        "\n4: Meat" \
        "\n5: Onions" \
        "\n6: Corn"
with open("menu.txt", 'w') as menu:
    menu.write(menuu)

class Pizza:
    def get_description(self):
        return self.description

    def get_cost(self):
        return self.cost

class Classic(Pizza):
    def get_description(self):
        return "Classic Pizza"

    def get_cost(self, size):
        return 7.25 + (size * 2)

class Margherita(Pizza):
    def get_description(self):
        return "Margherita Pizza"

    def get_cost(self, size):
        return 8.90 + (size * 2)

class Lahmacun(Pizza):
    def get_description(self):
        return "Turk Pizza (Lahmacun)"

    def get_cost(self, amount):
        return 3 * amount

class Pepperoni(Pizza):
    def get_description(self):
        return "Pepperoni Pizza"

    def get_cost(self, size):
        return 10 + (size * 2)

class FourSeasons(Pizza):
    def get_description(self):
        return "Four Seasons Pizza"

    def get_cost(self, size):
        return 10 + (size * 2)

class Decorator(Pizza):
    def __init__(self, component):
        self.component = component

    def get_cost(self):
        return self.component.get_cost() + Pizza.get_cost(self)

    def get_description(self):
        return self.component.get_description() + ' ' + Pizza.get_description(self)

class Olives(Decorator):
    def __init__(self, component):
        Decorator.__init__(self, component)
        self.description = "Olives"
        self.cost = 0.99

    def get_cost(self):
        return self.cost

    def get_description(self):
        return self.component.get_description() + ", " + self.description

class Mushrooms(Decorator):
    def __init__(self, component):
        Decorator.__init__(self, component)
        self.description = "Mushrooms"
        self.cost = 1.25

    def get_cost(self):
        return self.cost

    def get_description(self):
        return self.component.get_description() + ", " + self.description

class GoatCheese(Decorator):
    def __init__(self, component):
        Decorator.__init__(self, component)
        self.description = "Goat Cheese"
        self.cost = 1.25

    def get_cost(self):
        return self.cost

    def get_description(self):
        return self.component.get_description() + ", " + self.description

class Meat(Decorator):
    def __init__(self, component):
        Decorator.__init__(self, component)
        self.description = "Meat"
        self.cost = 1.99

    def get_cost(self):
        return self.cost

    def get_description(self):
        return self.component.get_description() + ", " + self.description

class Onions(Decorator):
    def __init__(self, component):
        Decorator.__init__(self, component)
        self.description = "Onions"
        self.cost = 0.99

    def get_cost(self):
        return self.cost

    def get_description(self):
        return self.component.get_description() + ", " + self.description

class Corn(Decorator):
    def __init__(self, component):
        Decorator.__init__(self, component)
        self.description = "Corn"
        self.cost = 0.99

    def get_cost(self):
        return self.cost

    def get_description(self):
        return self.component.get_description() + ", " + self.description

def get_pizza_choice():
    while True:
        print("Choose a pizza. İf you want to exit enter 'q' :")
        choice = input()
        if choice == "1":
            return Classic()
        elif choice == "2":
            return Margherita()
        elif choice == "3":
            return Lahmacun()
        elif choice == "4":
            return Pepperoni()
        elif choice == "5":
            return FourSeasons()
        elif choice == "q":
            return None
        else:
            print("Invalid choice!")

def get_size():
    choice = -1
    while choice == -1:
        print("Please choose size:\n"
              "0-Small\n"
              "1-Medium\n"
              "2-Large")
        try:
            choice = int(input())
        except:
            choice = -1
        if choice == 1 or choice == 2 or choice == 0:
            return choice
        else:
            print("İnvalid choice!")

def get_sauce_choice(pizza):
    choice = 0
    while True:
        print("1: Olives\n"
              "2: Mushrooms\n"
              "3: GoatCheese\n"
              "4: Meat\n"
              "5: Onions\n"
              "6: Corn")
        try:
            choice = int(input())
        except:
            pass
        if choice == 1:
            return Olives(pizza)
        elif choice == 2:
            return Mushrooms(pizza)
        elif choice == 3:
            return GoatCheese(pizza)
        elif choice == 4:
            return Meat(pizza)
        elif choice == 5:
            return Onions(pizza)
        elif choice == 6:
            return Corn(pizza)
        else:
            print("İnvalid selection")

def id_checker(id_number):
    if len(id_number) != 11 or not id_number.isdigit():
        print(id_number)
        print("Invalid ID number")
        return False
    else:
        return True

def get_last_credit_card(id_number):
    last_credit_card = ""
    try:
        with open("Orders_Database.csv", "r") as file:
            reader = csv.reader(file)
            for row in reader:
                if row[1] == id_number:
                    last_credit_card = row[2]
    except FileNotFoundError:
        pass
    return last_credit_card

def add_order_to_database(name, id_number, credit_card_number, pizza_description):
    with open("Orders_Database.csv", "a", newline="") as file:
        writer = csv.writer(file)
        now = datetime.datetime.now()
        order_time = now.strftime("%d-%m-%Y %H:%M:%S")
        writer.writerow([name, id_number, credit_card_number, pizza_description, order_time])

def credit_card_checker(credit_card):
    if len(credit_card) != 16 or not credit_card.isdigit():
        return False
    else:
        return True

if __name__ == '__main__':
    print(menuu)
    cost = 0.0
    pizza_choice = get_pizza_choice()
    if pizza_choice is not None:
        print(pizza_choice.get_description())
        c1 = input("Any sauce ?\nİf you want please enter '1'\n")
        if c1 == "1":
            souce = get_sauce_choice(pizza_choice)
            if pizza_choice.get_description() == "Turk Pizza (Lahmacun)":
                size = 0
                while size <= 0:
                    size = int(input("How many Turk Pizza do you want?\n"))
                    if size <= 0:
                        print("Invalid selection")
                cost = pizza_choice.get_cost(size) + souce.get_cost()
            else:
                size = get_size()
                cost = pizza_choice.get_cost(size) + souce.get_cost()
            if size == 0:
                size_text ="Small"
            elif size == 1:
                size_text ="Medium"
            else :
                size_text = "Large"

            description = souce.get_description() + ", " + size_text
            print(description)
            print("Cost : " + str(cost)+"$")

        else:
            size = get_size()
            if size == 0:
                size_text ="Small"
            elif size == 1:
                size_text ="Medium"
            else :
                size_text = "Large"
            cost = str(pizza_choice.get_cost(size))
            description = pizza_choice.get_description() + " ," + size_text
            print(description + " " + cost+"$")
        name = input("Name Surname :")
        id_number = input("Enter your ID number:\n")
        is_İd = id_checker(id_number)
        while True:
            if is_İd == False:
                id_number = input("Please check your ID number:\n")
                is_İd = id_checker(id_number)
            else:
                break

        last_credit_card = get_last_credit_card(id_number)

        if last_credit_card:
            use_last_credit_card = input("Do you want to use last credit card information? (Y/N): ")
            if use_last_credit_card.lower() == "y":
                credit_card_number = last_credit_card
            else:
                credit_card_number = input("Enter your credit card number: ")
                isCreditcard = credit_card_checker(credit_card_number)
                while True:
                    if not isCreditcard:
                        credit_card_number = input("Please check your Credit Card İnformation:\n")
                        isCreditcard = credit_card_checker(credit_card_number)
                    else:
                        break
        else:
            credit_card_number = input("Enter your credit card number: ")
            isCreditcard = credit_card_checker(credit_card_number)
            while True:
                if not isCreditcard:
                    credit_card_number = input("Please check your Credit Card İnformation:\n")
                    isCreditcard = credit_card_checker(credit_card_number)
                else:
                    break

        add_order_to_database(name, id_number, credit_card_number, description)

        print("Your order is taked, enjoy it :)")
    else:
        print("See you again ! ")


from .pages.product_page import ProductPage
from .pages.basket_page import BasketPage
from .pages.login_page import LoginPage
import pytest

link_product_page = "http://selenium1py.pythonanywhere.com/catalogue/coders-at-work_207"
link_product_page_another = "http://selenium1py.pythonanywhere.com/en-gb/catalogue/the-city-and-the-stars_95/"
link_user_product_page = "http://selenium1py.pythonanywhere.com/catalogue/coders-at-work_207"

@pytest.mark.need_review
@pytest.mark.parametrize('link', ["http://selenium1py.pythonanywhere.com/catalogue/coders-at-work_207/?promo=offer0",
                                   "http://selenium1py.pythonanywhere.com/catalogue/coders-at-work_207/?promo=offer1",
                                  "http://selenium1py.pythonanywhere.com/catalogue/coders-at-work_207/?promo=offer2",
                                  "http://selenium1py.pythonanywhere.com/catalogue/coders-at-work_207/?promo=offer3",
                                  "http://selenium1py.pythonanywhere.com/catalogue/coders-at-work_207/?promo=offer4",
                                  "http://selenium1py.pythonanywhere.com/catalogue/coders-at-work_207/?promo=offer5",
                                  "http://selenium1py.pythonanywhere.com/catalogue/coders-at-work_207/?promo=offer6",
                                  pytest.param("http://selenium1py.pythonanywhere.com/catalogue/coders-at-work_207/?promo=offer7", marks=pytest.mark.xfail),
                                  "http://selenium1py.pythonanywhere.com/catalogue/coders-at-work_207/?promo=offer8",
                                  "http://selenium1py.pythonanywhere.com/catalogue/coders-at-work_207/?promo=offer9"])
def test_guest_can_add_product_to_basket(browser, link):
    page = ProductPage(browser, link)
    page.open()
    page.add_product_to_basket()
    page.solve_quiz_and_get_code()
    page.should_be_add_to_basket_message()
    page.should_be_product_cost_equal_basket_cost()

@pytest.mark.xfail(reason="known message issue")
def test_guest_cant_see_success_message_after_adding_product_to_basket(browser):
    page = ProductPage(browser, link_product_page)
    page.open()
    page.add_product_to_basket()
    page.should_not_be_add_to_basket_message()

def test_guest_cant_see_success_message(browser):
    page = ProductPage(browser, link_product_page)
    page.open()
    page.should_not_be_add_to_basket_message_at_product_page()

@pytest.mark.xfail(reason="known message disappear issue")
def test_message_disappeared_after_adding_product_to_basket(browser):
    page = ProductPage(browser, link_product_page)
    page.open()
    page.add_product_to_basket()
    page.should_disappear_add_to_basket_message()

def test_guest_should_see_login_link_on_product_page(browser):
    page = ProductPage(browser, link_product_page_another)
    page.open()
    page.should_be_login_link()

@pytest.mark.need_review
def test_guest_can_go_to_login_page_from_product_page(browser):
    page = ProductPage(browser, link_product_page_another)
    page.open()
    page.go_to_login_page()

@pytest.mark.need_review
def test_guest_cant_see_product_in_basket_opened_from_product_page(browser):
    page = ProductPage(browser, link_product_page_another)
    page.open()
    page.go_to_basket_page()
    basket_page = BasketPage(browser, browser.current_url)
    basket_page.should_be_basket_empty()

@pytest.mark.user_work_with_basket
class TestUserAddToBasketFromProductPage():
    @pytest.fixture(scope="function", autouse=True)
    def setup(self, browser):
        link = "http://selenium1py.pythonanywhere.com/accounts/login/"
        page = LoginPage(browser, link)
        #prepare test data
        email_test_data, password_test_data = page.prepare_register_test_data()
        #setup step
        page.open()
        page.register_new_user(email_test_data, password_test_data)
        page.should_be_authorized_user()

    @pytest.mark.need_review
    def test_user_can_add_product_to_basket(self, browser):
        page = ProductPage(browser, link_user_product_page)
        page.open()
        page.add_product_to_basket()
        page.should_be_add_to_basket_message()
        page.should_be_product_cost_equal_basket_cost()

    def test_user_cant_see_success_message(self, browser):
        page = ProductPage(browser, link_user_product_page)
        page.open()
        page.should_not_be_add_to_basket_message_at_product_page()


import numpy as np
import pylab as plb
import matplotlib.pyplot as plt
import scipy
import scipy.signal
from scipy import optimize
from scipy import integrate
from scipy.optimize import curve_fit
from scipy import asarray as ar,exp
from scipy.interpolate import make_interp_spline
from scipy import stats
import os
import os.path
import ast
import glob

for f in glob.glob("Results\Peak Data\*"):
    os.remove(f)
for f in glob.glob("Results\Peak Plots\*"):       # Delete any previously saved results, to avoid potential
    os.remove(f)                                  # confusion between different data sets (filenames are not unique)
for f in glob.glob("Results\Raw Data\*"):
    os.remove(f)

cur_dir = os.getcwd() # Finds the current directory of where the python file is installed
back_dir = cur_dir + "\Background\\" + os.listdir(cur_dir + "\Background\\")[0] # Finds directory of background reading
meas_dir = cur_dir + "\Measured\\" + os.listdir(cur_dir + "\Measured\\")[0]     # and measurement reading

backtime = int(open(back_dir, "r").readline().strip())
meastime = int(open(meas_dir, "r").readline().strip())  # Pull the live times from both files

with open(back_dir, "r") as b:
    with open(meas_dir, "r") as m:
        countssource = [int(i) for i in m] # create a list for both the source and background reading
        countsback = [int(j) for j in b]

del countssource[0]
del countssource[0]
del countsback[0]
del countsback[0]    # remove the timestamps from the start of the files

if os.stat("MCA Channel Calibration\calibration.txt").st_size != 0:
    with open("MCA Channel Calibration\calibration.txt", "r") as file:
        calidata = file.read().splitlines()
    unit = "(keV)"
    calidata = list(map(float, calidata)) # obtains calibration data for MCA to keV units conversion, if available.
else:
    calidata = []
    unit = "(MCA Channels)"
if len(calidata) != 4:
    if len(calidata) != 0:
        raise ValueError("Incorrect Calibration Data")

peakssource = scipy.signal.find_peaks(countssource,height= 50, distance= 100)[0] # find any peaks in Source array.
# This function will give the indices of the central points of the peaks

ranger = 18
peakxrange = []    # Zooms into each individual peak, using "ranger" to define the range
peakyrange = []
for i in range(len(peakssource)):
    rmin = peakssource[i] - ranger
    if rmin < 0:
        rmin = 0
    rmax = peakssource[i] + ranger
    if rmax > (len(countssource) - 1):
        rmax = len(countssource) - 1            # takes the counts of each peak, and matches with corresponding
    if scipy.stats.normaltest(countssource[rmin:(rmax+1)])[1] < 0.045:    # MCA channels
        peakyrange.append(countssource[rmin:(rmax+1)])    # Applies a statistical test, so that only normally
        peakxrange.append(range(rmin,rmax+1))             # distributed datasets are accepted

backxrange = peakxrange
backyrange = []
for i in backxrange:                                    # For each accepted peak in the source counts, the corresponding
    backyrange.append([countsback[j] for j in i])       # background counts is found

def gauss(x, a, xc, w, y0):
    return a*exp(-(x-xc)**2/(2*abs(w)**2))+y0 # Define Gaussian function to be plotted. Parameters will be estimated

def linear(x, m, c):
    return m*x + c   # Define linear function, just incase a Gaussian cannot be fitted

for i in range(len(peakxrange)):

    x = peakxrange[i]
    xc = peakxrange[i][scipy.signal.find_peaks(peakyrange[i],distance= 36)[0][0]]
    w = scipy.signal.peak_widths(peakyrange[i], scipy.signal.find_peaks(peakyrange[i],distance= 36)[0], 0.5)[0][0]
    FWTM = scipy.signal.peak_widths(peakyrange[i], scipy.signal.find_peaks(peakyrange[i], distance=36)[0], 0.05)[0][
        0]  # Define guesses for parameters of Gaussian

    wback = scipy.signal.peak_widths(backyrange[i], scipy.signal.find_peaks(backyrange[i],distance= 36)[0], 0.5)[0][0]
    FWTMback = scipy.signal.peak_widths(backyrange[i], scipy.signal.find_peaks(backyrange[i], distance=36)[0], 0.05)[0][0]

    xback = x
    xcback = xc

    prom = scipy.signal.peak_prominences(peakyrange[i], scipy.signal.find_peaks(peakyrange[i],distance= 36)[0])
    promback = scipy.signal.peak_prominences(backyrange[i], scipy.signal.find_peaks(
        scipy.signal.savgol_filter(backyrange[i],3,2),distance= 36)[0])  # finds the height of each peak, and the
                                                                         # positions of the peak bases
    h = prom[0][0]
    hback = promback[0][0]
    y = peakyrange[i]
    el = scipy.signal.peak_widths(peakyrange[i], scipy.signal.find_peaks(peakyrange[i],distance= 36)[0], 1)[1][0]
    yback = backyrange[i]
    elback = scipy.signal.peak_widths(backyrange[i], scipy.signal.find_peaks(peakyrange[i],distance= 36)[0], 1)[1][0]

    try:

        popt, pcov = optimize.curve_fit(gauss, x, y, [h, xc, w, el], method="trf")  # Fit a curve to each peak
        try:
            poptback, pcovback = optimize.curve_fit(gauss, xback, yback, [hback, xcback, wback, elback], method="trf")
            func = 0
            perrback = np.sqrt(np.diag(pcovback))
            netcountserrback = np.sqrt(np.pi * ((poptback[2] * perrback[0]) ** 2 + (poptback[0] * perrback[2]) ** 2 + 2
                                                * poptback[0] * poptback[2] * pcovback[0][2]))
            if netcountserrback > 2000:
                poptback, pcovback = optimize.curve_fit(linear, xback, yback, [0, elback], method="trf")
                func = 1
        except:
            poptback, pcovback = optimize.curve_fit(linear, xback, yback, [0, elback], method="trf")
            func = 1

        perr = np.sqrt(np.diag(pcov))
        perrback = np.sqrt(np.diag(pcovback))

        peakintegral = integrate.quad(lambda t: gauss(t, *popt), x[prom[1][0]], x[prom[2][0]])
        netcounts = peakintegral[0]
        netcountserr = np.sqrt(2 * np.pi * ((popt[2] * perr[0]) ** 2 + (popt[0] * perr[2]) ** 2 + 2 * popt[0] * popt[2]
                                           * pcov[0][2]))

        if func == 0:
            peakintegralback = integrate.quad(lambda t: gauss(t, *poptback), x[prom[1][0]], x[prom[2][0]])
            netcountserrback = np.sqrt(np.pi * ((poptback[2] * perrback[0]) ** 2 + (poptback[0] * perrback[2]) ** 2 + 2
                                                * poptback[0] * poptback[2] * pcovback[0][2]))
        else:
            peakintegralback = integrate.quad(lambda t: linear(t, *poptback), x[prom[1][0]], x[prom[2][0]])
            netcountserrback = np.sqrt((perrback[0]/2) ** 2 + (perrback[1] ** 2))

        netcountsback = peakintegralback[0]  # Integrate each peak to find the net counts

        yplot = [s - (l * meastime / backtime) for s, l in zip(peakyrange[i], backyrange[i])]
        FWHM = scipy.signal.peak_widths(yplot, scipy.signal.find_peaks(peakyrange[i],distance= 36)[0], 0.5)[0][0]

        with open("Results\Peak Data\peak " + str(i + 1) + ".txt", "w") as file:
            if len(calidata) != 0:  # background is normalised, and subtracted from the counts
                file.write("Net Counts:            " + str((netcounts - netcountsback*meastime/backtime))
                        + " +/- " + str(np.sqrt( netcountserr**2 + (netcountserrback*meastime/backtime)**2 )
                                        ) + " (Counts)")
                file.write("\nNet Count Rate:        " + str((netcounts - netcountsback * meastime / backtime)/meastime)
                           + " +/- " + str(np.sqrt(netcountserr ** 2 + (netcountserrback * meastime / backtime) ** 2)
                                           /meastime) + " (Counts/s)")
                file.write("\nPeak Height:           " + str(popt[0]) + " +/- " + str(perr[0]) + " (Counts)")
                file.write("\nPeak Central Position: " + str(popt[1]*calidata[0] + calidata[2]) + " +/- " + str(
                    np.sqrt(perr[1] ** 2 + calidata[3] ** 2)) + " " + unit)
                file.write("\nPeak FWHM:             " + str(FWHM*calidata[0]) + " +/- " + str(
                    np.sqrt(perr[2] ** 2 + calidata[3] ** 2)) + " " + unit)
            else:
                file.write("Net Counts:            " + str(netcounts - netcountsback*meastime/backtime) + " +/- " +
                        str(np.sqrt( netcountserr**2 + (netcountserrback*meastime/backtime)**2 )) + " (Counts)")
                file.write("\nNet Count Rate:        " + str((netcounts - netcountsback * meastime / backtime)/meastime)
                           + " +/- " + str(np.sqrt(netcountserr ** 2 + (netcountserrback * meastime / backtime) ** 2)
                                           /meastime) + " (Counts/s)")
                file.write("\nPeak Height:           " + str(popt[0]) + " +/- " + str(perr[0]) + " (Counts)")
                file.write("\nPeak Central Position: " + str(popt[1]) + " +/- " + str(perr[1]) + " " + unit)
                file.write("\nPeak FWHM:             " + str(FWHM) + " +/- " + str(perr[2]) + " " + unit)
        with open("Results\Raw Data\peak " + str(i + 1) + ".txt", "w") as file:
            for (x1, y1) in zip(x, yplot):
                file.write("{0}\t{1}\n".format(x1, y1))  # Saves all useful data from each peak

        if len(calidata) != 0:
            x = [j*calidata[0] + calidata[2] for j in x]
            xc = xc*calidata[0] + calidata[2]
            w = FWHM*calidata[0]

        poptplot, pcovplot = optimize.curve_fit(gauss, x, yplot, [h, xc, w, el], method="trf") # creates a new dataset,
        # with subtracted normalised background, so that a representation of each peak can be shown
        xsmooth = np.linspace(x[0], x[len(x) - 1], len(x) * 100)
        smooth = make_interp_spline(x, yplot, k=3)
        ysmooth = smooth(xsmooth) # Uses fitted function to create more data points, to plot a smooth line

        plt.plot(x, yplot, "b+", label="Counts")
        plt.plot(xsmooth, gauss(xsmooth, *poptplot), "k:", label="Best Fit")
        plt.legend()
        plt.xlabel("Energy " + unit)
        plt.ylabel("Counts")
        plt.savefig("Results\Peak Plots\Fig." + str(i + 1) + " - Full Energy Peak.png")
        plt.show() # Plots and saves peak

    except:
        pass  # If any of the fitting process fails due to "false peaks", the program will move on without stopping


"""empty message

Revision ID: 92be4067e5bf
Revises: 6c9a4dfcd77b
Create Date: 2020-08-24 21:11:35.660453

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '92be4067e5bf'
down_revision = '6c9a4dfcd77b'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_constraint('comments_user_id_fkey', 'comments', type_='foreignkey')
    op.drop_column('comments', 'user_id')
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('comments', sa.Column('user_id', sa.INTEGER(), autoincrement=False, nullable=False))
    op.create_foreign_key('comments_user_id_fkey', 'comments', 'users', ['user_id'], ['id'])
    # ### end Alembic commands ###

Flask API - Social Media Backend/migrations/versions/92be4067e5bf_.py


from django.db import models
from django.core.validators import MinValueValidator, MaxValueValidator
# Create your models here.

class Movie(models.Model):
    title = models.CharField(max_length=20)
    audience = models.IntegerField()
    release_date = models.DateField()
    genre_menu = (('코미디','코미디'),('공포','공포'),('로맨스','로맨스'))
    genre = models.CharField(max_length=30,choices=genre_menu)
    score = models.IntegerField(validators=(MinValueValidator(0), MaxValueValidator(5),))
    poster_url = models.CharField(max_length=50)
    description = models.TextField()
    actor_image = models.ImageField(blank=True, null=True)


import cv2
import numpy as np

# Read the image
img = cv2.imread('lena.jpg')

# Convert the image to grayscale
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

# Apply Gaussian blur to the image
blur = cv2.GaussianBlur(gray, (5,5), 0)

# Perform unsharp masking
unsharp = cv2.addWeighted(gray, 1.5, blur, -0.5, 0)

# Perform Laplacian domain frequency filtering
laplacian = cv2.Laplacian(gray, cv2.CV_64F)
laplacian_filtered = np.uint8(np.abs(laplacian))

# Perform selective filtering
sel_filter = np.zeros_like(gray)
sel_filter[150:250, 150:250] = 1
sel_filtered = cv2.bitwise_and(gray, gray, mask=sel_filter)

# Display the results
cv2.imshow('Original Image', img)
cv2.imshow('Unsharp Masking', unsharp)
cv2.imshow('Laplacian Domain Frequency Filtering', laplacian_filtered)
cv2.imshow('Selective Filtering', sel_filtered)
cv2.waitKey(0)
cv2.destroyAllWindows()


import numpy as np
from qiskit.quantum_info import Statevector

class VQE:
	def __init__(self, ansatz, optimizer):
		self.ansatz = ansatz
		self.optimizer = optimizer

		self.hamiltonian = None
		self.x0 = None
		self.iter = None
		self.cost = None
		self.result = None

	def run(self, hamiltonian, x0=None):
		self.hamiltonian = hamiltonian
		if x0 is None:
			x0 = np.random.uniform(-2 * np.pi, 2 * np.pi, self.ansatz.num_parameters)
		self.x0 = x0

		self.iter = 0

		self.result = self.optimizer.minimize(self.cost_fun, self.x0)

		return VQEResult(self)

	def cost_fun(self, x):
		self.iter += 1
		qc = self.ansatz.assign_parameters(x)
		self.cost = Statevector(qc).expectation_value(self.hamiltonian).real
		self.callback()
		return self.cost

	def callback(self):
		if self.iter % 100 == 0:
			print(f'{self.iter}', f'{self.cost:.8f}')

class VQEResult:
	def __init__(self, vqe):
		self.eigenvalue = vqe.result.fun
		self.eigenstate = Statevector(vqe.ansatz.assign_parameters(vqe.result.x)).data
		self.optimal_parameters = vqe.result.x


import sys
import gym
import time
from gym import wrappers
from gym_foo.agent.agent import Agent
from gym_foo.agent.random_agent import RandomAgent
from gym_foo.agent.q_learning_agent_v2 import QLearnAgent_V2
from gym_foo.agent.attack_agent import TowerRushAgent, AttackTowerAgent, AttackMainTowerAgent
from gym_foo.model.battlefield_detail import BattleField
from gym_foo.Utils import *
import random
import os

object_path = os.path.dirname(__file__)
alpha = 0.5
gamma = 0.9
agent_model_path = os.path.join(object_path, 'agent_model_v2/train_0/agent0.pkl')

if __name__ == "__main__":
    win = 0
    total = 0
    agent_0 = QLearnAgent_V2(team=0, agent_model_path=agent_model_path)
    for iter in range(500):
        grid = gym.make('ava-v1')
        # grid = wrappers.Monitor(grid, './videos', force=True)  # 记录回放动画
        grid.reset()
        # grid.render()
        is_start = True
        agent_1 = random.choice([TowerRushAgent(team=1), AttackTowerAgent(team=1), AttackMainTowerAgent(team=1)])
        while True:
            if is_start:
                state, score, is_terminate, detail = grid.initial_state()
                is_start = False

            detail_for_agent0 = BattleField(
                battle_field_total=detail,
                team=0
            )

            detail_for_agent1 = BattleField(
                battle_field_total=detail,
                team=1
            )

            agent0_s0_list = []
            for hive in detail_for_agent0.get_self_hives():
                agent0_s0_list.append(get_status_rewards_from_detail_v2(detail_for_agent0, hive)[1])
            agent1_r11, agent1_s0 = get_status_rewards_from_detail(detail_for_agent1)
            agent0_a0_list, agent0_action_list = agent_0.generate_action(state, score, detail_for_agent0)
            action_list = [agent0_action_list, agent_1.generate_action(state, score, detail_for_agent1)]
            state, score, is_terminate, detail = grid.step(action_list)
            s1_detail_for_agent0 = BattleField(
                battle_field_total=detail,
                team=0
            )
            s1_detail_for_agent1 = BattleField(
                battle_field_total=detail,
                team=1
            )
            agent0_s1_list = []
            agent0_r0 = None
            for hive in detail_for_agent0.get_self_hives():
                agent0_r0, agent0_s1 = get_status_rewards_from_detail_v2(s1_detail_for_agent0, hive)
                agent0_s1_list.append(agent0_s1)
            agent1_r0, agent1_s1 = get_status_rewards_from_detail(s1_detail_for_agent1)
            # train agent0
            # s1处的最大动作
            a1_list = [agent_0.greedy(agent0_s1) for agent0_s1 in agent0_s1_list]
            key1 = ["%s_%s" % (agent0_s1, a1) for agent0_s1, a1 in zip(agent0_s1_list, a1_list)]
            key = ["%s_%s" % (agent0_s0, agent0_a0) for agent0_s0, agent0_a0 in zip(agent0_s0_list, agent0_a0_list)]
            # 利用qlearning方法更新值函数

            for k, k1 in zip(key, key1):
                agent_0.qfunc[k] = agent_0.qfunc.get(k, 0.0) + alpha * (agent0_r0 + gamma * agent_0.qfunc.get(k1, 0.0) - agent_0.qfunc.get(k, 0.0))

            if is_terminate:
                agent_0.save_agent_model(agent_model_path)
                # print('Over, score:', score, detail)
                s0, s1 = detail.get_score()
                if s0 >= s1:
                    win += 1
                total += 1
                print('iter: %d, score: %s, win: %d, total: %d, win_rate: %f, beat: %s, state_num: %d' % (iter, str(detail.get_score()), win, total, win / total, type(agent_1), len(agent_0.qfunc)))
                # print(detail.get_battle_reports())
                # print(detail_for_agent0.get_scout_reports())
                # print(detail_for_agent1.get_scout_reports())
                break

            else:
                # print('Round, score:', score, detail)
                pass


import pandas as pd

from datetime import datetime

from general_utils import write_log

def load_data_to_postgresql(db_handlers, df_db_books, df_db_people, df_db_records):
    pg_cursor = db_handlers[2]
    pg_conn = db_handlers[3]
    write_log('Starting loading data into PostgreSQL database')
    write_log('Loading books in PostgreSQL')
    for book in df_db_books:
        try:
            temp1 = book['book_num_1']
            temp2 = book['book_num_2']
            if(pd.isnull(temp1)):
                temp1 = 0
            if(pd.isnull(temp2)):
                temp2 = 0
            temp1 = (int)(temp1)
            temp2 = (int)(temp2)
            # Добавить ON CONFLICT
            pg_cursor.execute("INSERT INTO books (id_book,title_book,author_book, type_book, year_book, library1_code, library2_code) " +\
                              "VALUES (%s, %s, %s, %s, %s, %s, %s) " +\
                              "ON CONFLICT(id_book) " +\
                              "DO UPDATE SET (library1_code, library2_code)=(EXCLUDED.library1_code, EXCLUDED.library2_code)",
                              (book['id'],book['title'],book['author'],book['type'],book['year'],temp1,temp2)
                              )
            pg_conn.commit()
        except Exception as e:
            print('Error ' + str(e))
            write_log('Error loading books in PostgreSQL: ' + str(e))
    write_log('Finished loading books')
    write_log('Starting loading people in PostgreSQL')
    for person in df_db_people:
        try:
            temp1 = person['library_card_1']
            temp2 = person['library_card_2']
            if(pd.isnull(temp1)):
                temp1 = 0
            if(pd.isnull(temp2)):
                temp2 = 0
            temp1 = (int)(temp1)
            temp2 = (int)(temp2)
            pg_cursor.execute("INSERT INTO people (id,surname_reader, name_reader, patronymic_reader, date_of_birth, library1_code, library2_code) " +\
                              "VALUES (%s, %s, %s, %s, %s, %s, %s) " +\
                              "ON CONFLICT (id) " +\
                              "DO UPDATE SET (library1_code, library2_code)=(EXCLUDED.library1_code, EXCLUDED.library2_code) ",
                              (person['id'],
                               person['surname'],
                               person['name'],
                               person['patronymic'],
                               person['date_of_birth'],
                               temp1, temp2)
                              )
            pg_conn.commit()
        except Exception as e:
            print('Error '+str(e))
            write_log('Error loading people in PostgreSQL ' + str(e))
    try:
        pg_cursor.executemany("INSERT INTO records (id_reader, id_book, date_of_issue, date_of_return, id_library) " +\
                              "VALUES (%s, %s, %s, %s, %s)", df_db_records)
        pg_conn.commit()
    except Exception as e:
        write_log('Error loading records in PostgreSQL ' + str(e))

def load_data_to_clickhouse(db_handlers,date_of_extraction):
    pg_cursor = db_handlers[2]
    ch_conn = db_handlers[4]
    temp_key = datetime.strptime(date_of_extraction, '%d/%m/%Y')
    key_for_postgres = temp_key.strftime('%Y-%m-%d')
    pg_cursor.execute("SELECT id_rec, id_reader,id_book, (SELECT type_book from books where id_book = records.id_book), date_of_issue, date_of_return, id_library " +\
                      "FROM records WHERE CAST(date_of_issue as DATE) = "
                      + str("'"+key_for_postgres+"'"))
    pg_data = pg_cursor.fetchall()
    ch_data = [(int(row[0]),int(row[1]),int(row[2]),row[3],row[4],row[5],int(row[6])) for row in pg_data]
    ch_conn.execute("INSERT INTO recdata2 " +\
                    "(id_rec, id_reader, id_book, type_book, date_of_issue, date_of_return, lib_id) VALUES", ch_data)