"""relative.py -- generation and manipulation of relative matrix elements

Patrick J. Fasano
University of Notre Dame

- 03/20/22 (pjf): Created.
- 06/29/22 (pjf): Initial implementation completed.
- 07/06/22 (pjf): Add save_rel() and save_moshinsky().
"""
from __future__ import annotations
import glob
import os
from typing import Callable

import mcscript.control
import mcscript.utils
import mcscript.exception
import mcscript.task

from . import (
    utils,
    modes,
    environ,
    operators,
)

_k_rel_target_handlers:dict[str,Callable] = {}

def relative_filename_for_source(source_id:str, source:dict) -> str:
    """Generate relative filename for given source defintion.

    Arguments:
        source (dict): relative/relative-cm source as defined in operators.rel

    Returns:
        (str): filename for relative/relative-cm source
    """
    # if the source is a file, find it on the filesystem
    if source["source_type"] == "file":
        filename = source.get("parameters", {}).get("filename")
        if filename is not None:
            rel_filename = mcscript.utils.expand_path(filename)
            if not os.path.isfile(rel_filename):
                rel_filename = mcscript.utils.search_in_subdirectories(
                    environ.data_dir_rel_list, environ.rel_dir_list,
                    filename, fail_on_not_found=True
                )
        else:
            rel_filename = environ.find_rel_file(
                source_id, source["Nmax"], source["hw"]
            )
    # otherwise, find the filename we will use for output
    else:
        if source["type"] == operators.rel.RelativeOperatorType.kRelative:
            rel_filename = environ.rel_filename(source_id, source['Nmax'], source['hw'])
        elif source["type"] == operators.rel.RelativeOperatorType.kRelativeCM:
            rel_filename = environ.relcm_filename(source_id, source['Nmax'], source['hw'])
        else:
            raise ValueError("invalid operator type: {}".format(source["type"]))

    return rel_filename

_k_rel_target_handlers["file"] = (
    lambda source_id, sources: relative_filename_for_source(source_id, sources[source_id])
)

def _jpv2rel_handler(source_id:str, rel_sources:dict):
    source = rel_sources[source_id]
    parameters = source["parameters"]
    qn = source["qn"]
    (J0, g0, T0_min, T0_max) = qn
    if (J0 != 0) or (g0 != 0) or (T0_min != 0):
        raise mcscript.exception.ScriptError(f"invalid quantum numbers for jpv2rel: {qn}")
    rel_filename = relative_filename_for_source(source_id, source)

    lines = []
    lines.append(f"{source['Nmax']} {parameters['Jmax']} {T0_max:d}")
    if T0_max == 0:
        lines.append(parameters["source_filename"])
    else:
        lines.append('{source_filename_pp:s} {source_filename_nn:s} {source_filename_pn:s}'.format(**parameters))
    lines.append(rel_filename)
    lines += [""]

    mcscript.control.call(
        [
            environ.shell_filename("jpv2rel")
        ],
        input_lines=lines,
        mode=mcscript.control.CallMode.kSerial,
        check_return=True
    )

    return rel_filename

_k_rel_target_handlers["jpv"] = _jpv2rel_handler

_k_relativegen_operator_patterns = {
    "coordinate-sqr": " {coordinate_type:s} {T0:d}",
    "dipole": " {coordinate_type:s} {T0:d}",
    "quadrupole": " {coordinate_type:s} {T0:d}",
    "orbital-am": " {T0:d}",
    "spin-am": " {T0:d}",
    "coulomb": " {species:s} {steps:d}",
    "symmunit": " {T0:d} {Np:d} {Lp:d} {Sp:d} {Jp:d} {Tp:d} {N:d} {L:d} {S:d} {J:d} {T:d}",
    "interaction": " {interaction:s} {parameters:s}",
    "LENPIC-N2LOGT": " {regulator:g} {oscillator_length:15.8e} {steps:d}",
    "LENPIC-NLOM1": " {regulator:g} {oscillator_length:15.8e} {steps:d}",
}
def _relativegen_handler(source_id:str, rel_sources:dict):
    source = rel_sources[source_id]
    parameters = source["parameters"]
    qn = source["qn"]
    (J0, g0, T0_min, T0_max) = qn
    operator_name = parameters["operator_name"]
    rel_filename = relative_filename_for_source(source_id, source)

    lines = []
    lines += [f"{J0:d} {g0:d} {T0_min:d} {T0_max:d}"]
    lines += [f"{source['Nmax']:d}"]
    lines += [operator_name + _k_relativegen_operator_patterns.get(operator_name, "").format(**parameters)]
    lines += [rel_filename]
    lines += [""]

    mcscript.control.call(
        [
            environ.shell_filename(source["source_type"])
        ],
        input_lines=lines,
        mode=mcscript.control.CallMode.kSerial,
        check_return=True
    )

    return rel_filename

_k_rel_target_handlers["relative-gen"] = _relativegen_handler
_k_rel_target_handlers["relcm-gen"] = _relativegen_handler

def _relativexform_handler(source_id:str, rel_sources:dict):
    source = rel_sources[source_id]
    parameters = source["parameters"]
    input_source_id = source["inputs"][0]
    input_source = rel_sources[input_source_id]
    if source["qn"] != input_source["qn"]:
        raise mcscript.exception.ScriptError(
            "incompatible quantum numbers: {} {}".format(source["qn"],input_source["qn"])
        )
    rel_filename = relative_filename_for_source(source_id, source)

    # TODO: update once relative-xform is generalized
    if source["type"] != operators.rel.RelativeOperatorType.kRelative:
        raise ValueError("invalid operator type: {}".format(source["type"]))

    input_filename = relative_filename_for_source(input_source_id, input_source)

    b_ratio = utils.oscillator_length(source["hw"])/utils.oscillator_length(input_source["hw"])

    lines = []
    lines += [input_filename]
    lines += [f"{source['Nmax']:d} {b_ratio:.17e} {parameters['steps']:d}"]
    lines += [rel_filename]
    lines += [""]

    mcscript.control.call(
        [
            environ.shell_filename("relative-xform")
        ],
        input_lines=lines,
        mode=mcscript.control.CallMode.kSerial,
        check_return=True
    )

    return rel_filename

_k_rel_target_handlers["relative-xform"] = _relativexform_handler

def generate_rel_targets(task):
    """Generate relative and relative-cm matrix elements for task.

    Arguments:
        task (dict): as described in module docstring
    """
    # get targets
    targets = operators.rel.get_rel_targets(task)

    # get rel sources
    rel_sources = operators.rel.get_rel_sources(task, targets)

    # dispatch to handlers
    for source_id,source in rel_sources.items():
        _k_rel_target_handlers[source["source_type"]](source_id, rel_sources)

def generate_moshinsky_targets(task):
    """Perform Talmi-Moshinsky transformation for task.

    Arguments:
        task (dict): as described in module docstring
    """
    targets = task.get("moshinsky_targets", [])

    # get rel sources
    rel_sources = operators.rel.get_rel_sources(task, {target[2]["id"] for target in targets})

    for (name, qn, parameters) in targets:
        (J0, g0, Tz0) = qn
        relative_id = parameters["id"]
        relative_source = rel_sources[relative_id]
        truncation = parameters["truncation"]
        rel_filename = relative_filename_for_source(relative_id, relative_source)
        tbme_filename = environ.tmbe_filename(
            name, truncation, relative_source['hw'], ext=task["h2_extension"]
        )

        if ((J0 != relative_source["qn"][0])
            or (g0 != relative_source["qn"][1])
            or (Tz0 > relative_source["qn"][3])
        ):
            raise mcscript.exception.ScriptError(
                f"incompatible quantum numbers: {relative_source['qn']} {qn}"
            )

        lines = []
        lines += ["{:s} {:d}".format(*truncation)]
        lines += [f"{rel_filename:s} {relative_source['type'].value:s}"]
        lines += [f"{tbme_filename:s} jjjpn {task['h2_format']} {Tz0:d}"]
        lines += [""]

        mcscript.control.call(
            [
                environ.shell_filename("moshinsky")
            ],
            input_lines=lines,
            mode=mcscript.control.CallMode.kSerial,
            check_return=True
        )

def save_rel(task):
    """Save relative targets to results.

    Arguments:
        task (dict): as described in module docstring
    """
    # get targets
    targets = operators.rel.get_rel_targets(task)

    # get rel sources
    rel_sources = operators.rel.get_rel_sources(task, targets)

    for (basename, _, source_id) in task.get("relative_targets", []):
        source = rel_sources[source_id]
        source_filename = relative_filename_for_source(source_id, source)
        if source["type"] == operators.rel.RelativeOperatorType.kRelative:
            target_filename = environ.rel_filename(basename, source['Nmax'], source['hw'])
        elif source["type"] == operators.rel.RelativeOperatorType.kRelativeCM:
            target_filename = environ.relcm_filename(basename, source['Nmax'], source['hw'])
        else:
            raise ValueError("invalid operator type: {}".format(source["type"]))
        mcscript.task.save_results_single(
            task, source_filename, target_filename, subdirectory="rel"
        )

def save_moshinsky(task):
    """Save moshinsky targets to results.

    Arguments:
        task (dict): as described in module docstring
    """
    # get targets
    targets = operators.rel.get_rel_targets(task)

    # get rel sources
    rel_sources = operators.rel.get_rel_sources(task, targets)

    for (basename, _, parameters) in task.get("moshinsky_targets", []):
        source_id = parameters["id"]
        source = rel_sources[source_id]
        filename = environ.tmbe_filename(
            basename, parameters["truncation"], source['hw'], task.get("h2_extension", "bin")
        )
        mcscript.task.save_results_single(
            task, filename, subdirectory="tbme"
        )


import logging
import os
import re
import sys
import git

from .mixin import ComparableMixin
from .error import UnresolvedConflictsError

class Branch(ComparableMixin):
    _regex_minor = re.compile(r'(\d+)\.(\d+)')
    _branch = None

    _v_major = None

    @classmethod
    def create(cls, github_branch):
        if github_branch.name.isdecimal():  # it's major
            return MajorBranch(int(github_branch.name), github_branch)

        else:
            match = cls._regex_minor.fullmatch(github_branch.name)
            if not match:
                return None

            version = match.groups()
            return MinorBranch(int(version[0]), int(version[1]), github_branch)

    def __init__(self, github_branch):
        self._branch = github_branch

    def sameMajor(self, other):
        return self._v_major == other._v_major

    @property
    def name(self):
        return self._branch.name

class MajorBranch(Branch):
    def __init__(self, version, github_branch):
        super().__init__(github_branch)
        self._v_major = version

    def _cmpkey(self):
        return (self._v_major, sys.maxsize)

class MinorBranch(Branch):
    _v_minor = None

    def __init__(self, major_version, minor_version, github_branch):
        super().__init__(github_branch)
        self._v_major = major_version
        self._v_minor = minor_version

    def _cmpkey(self):
        return (self._v_major, self._v_minor)

class ProxyRepo:
    _target = None
    _github_repo = None
    _git_repo = None
    _branches = []
    _skip = []

    def __init__(self, target):
        self._target = target
        self._github_repo = target.github_repo

        _path = '/tmp/ss-merger-{}'.format(self._github_repo.id)

        logging.info('Checking out into %s', _path)

        if os.path.exists(_path):
            logging.warning('Folder already exists, skipping')
            self._git_repo = git.Repo(_path)
        else:
            self._git_repo = git.Repo.clone_from(self._github_repo.clone_url, _path)
            logging.info('Done')

        for github_branch in self._github_repo.get_branches():
            branch = Branch.create(github_branch)
            if not branch:
                continue
            if target.config('only') and str(branch._v_major) not in [str(v) for v in target.config('only')]:
                continue
            if branch.name in target.config('skip', []):
                continue
            self._branches.append(branch)

        self._branches.sort()
        logging.info('Identified merge-up branches: %s', [b.name for b in self._branches])

        self._git_repo_init()

    @property
    def branches(self):
        return self._branches

    @property
    def g_repo(self):
        return self._git_repo

    @property
    def gh_repo(self):
        return self._github_repo

    def _git_repo_init(self):
        origin = self._git_repo.remotes.origin
        origin.fetch()

        for branch in self.branches:
            try:
                self._git_repo.create_head(branch.name, origin.refs[branch.name].commit)
            except:
                pass

            if self._git_repo.heads[branch.name].commit != origin.refs[branch.name].commit:
                self._skip.append(branch.name)
                logging.warning('Skipping %s (different with origin; already merged-up?)', branch.name)

        if 'mergeup_fork' in self._git_repo.remotes:
            logging.info('Mergeup remote is %s', [i for i in self._git_repo.remote('mergeup_fork').urls][0])
        else:
            logging.info('Mergeup remote is %s', self._target.fork.github_repo.ssh_url)
            self._git_repo.create_remote('mergeup_fork', self._target.fork.github_repo.ssh_url)

    def mergeUp(self, conflict_resolvers):
        # import ipdb; ipdb.set_trace()
        idx = 1

        while len(self.branches) > idx+1:
            self._mergeUp(idx, conflict_resolvers)
            idx += 1

    def _mergeUp(self, idx, conflict_resolvers):
        src_branch = self.branches[idx]
        dst_branch = self.branches[idx+1]

        src = src_branch.name
        dst = dst_branch.name

        if dst in self._skip:
            return True;

        if not self._target.config('merge-majors', False) and not src_branch.sameMajor(dst_branch):
            logging.warning('Skipping merge {} to {}'.format(src, dst))
            return True

        mergeup_commit_message = 'Merge-up "{}" into "{}"'.format(src, dst)

        logging.info('%s', mergeup_commit_message)

        src_head = self._git_repo.heads[src]
        dst_head = self._git_repo.heads[dst]
        dst_head.checkout()

        dst_original_commit = dst_head.commit

        try:
            self._git_repo.git.merge(src_head, message=mergeup_commit_message, commit=True)

        except git.exc.GitCommandError as e:
            logging.warning('Conflict! %s', e.stdout)

            if len(self._git_repo.index.unmerged_blobs()):
                self._resolveConflicts(conflict_resolvers)
            else:
                raise Exception("Not sure what's wrong...")

            if len(self._git_repo.index.unmerged_blobs()):
                raise UnresolvedConflictsError()
                # raise Exception('Unresolved conflicts!')

            self._git_repo.git.commit(message=mergeup_commit_message)

        dst_mergeup_commit = self._git_repo.commit()

        if dst_mergeup_commit != dst_original_commit:
            self._pushMergedBranch(dst)
            self._pullRequestMergedBranch(dst)
        else:
            logging.info('Already up-to-date')

    def _resolveConflicts(self, conflict_resolvers):
        for resolver in conflict_resolvers:
            if resolver.isKnown(self._git_repo):
                try:
                    logging.warning('Running %s', resolver.name)
                    resolver.resolve(self._git_repo)
                except:
                    logging.exception('Resolver Failed')

    def _pushMergedBranch(self, dst_name):
        the_branch = 'merge-up/{}'.format(dst_name)

        self._git_repo.git.checkout(b=the_branch)

        logging.warning('Pushing %s to %s', the_branch, [i for i in self._git_repo.remote('mergeup_fork').urls][0])

        self._git_repo.git.push('mergeup_fork', the_branch)

        return True

    def _pullRequestMergedBranch(self, dst_name):
        the_branch = 'merge-up/{}'.format(dst_name)

        logging.warning('Creating a PR for %s', the_branch)

        self._target.fork.github_repo.create_pull(
            title=self._git_repo.branches[dst_name].commit.message,
            body='Created by the auto-merger ([RFC 9083](https://github.com/silverstripe/silverstripe-framework/issues/9083))',
            head=the_branch,
            base=dst_name
        )


#!/usr/bin/python3.7
# 剑指 Offer 31. 栈的压入、弹出序列
from typing import List
class Solution:
    def validateStackSequences(self, pushed: List[int], popped: List[int]) -> bool:
        stack,index = [],0
        for i in range(len(pushed)):
            stack.append(pushed[i])
            while stack and popped[index] == stack[-1]:
                stack.pop()
                index += 1
        return not stack

if __name__ == '__main__':

    try:
        assert exec
        print('解答正确')
    except AssertionError:
        print('解答错误')

剑指 Offer 31. 栈的压入、弹出序列.py


import tensorflow as tf
import numpy as np
import time
import argparse
import model
import math
import utils
import data
import csv

from utils import Option
opt = Option('./config.json')

utils.init()

formatter = argparse.ArgumentDefaultsHelpFormatter
parser = argparse.ArgumentParser(formatter_class=formatter)

args, flags = utils.parse_args(opt, parser)

tf.compat.v1.random.set_random_seed(args['random_seed'])

def rampup(epoch):
    if epoch < args['rampup_length']:
        p = max(0.0, float(epoch)) / float(args['rampup_length'])
        p = 1.0 - p
        return math.exp(-p * p * 5.0)
    else:
        return 1.0

def rampup_ratio(epoch):
    if epoch < args['rampup_ratio']:
        p = max(0.0, float(epoch)) / args['rampup_ratio']
        p = 1.0 - p
        return math.exp(-p * p * 5.0)
    else:
        return 1.0

def rampdown(epoch):
    if epoch >= (args['n_epochs'] - args['rampdown_length']):
        ep = (epoch - (args['n_epochs'] - args['rampdown_length'])) * 0.5
        return math.exp(-(ep * ep) / args['rampdown_length'])
    else:
        return 1.0

args['shape'] = (28, 28, 1) if args['dataset'] == 'mnist' else (32, 32, 3)

BAE   = model.BAE(args)
mnist = data.mnist(args)

unlabeled_idx = np.copy(mnist.unlabeled_idx)
labeled_idx = np.copy(mnist.labeled_idx)

sparse_label  = np.copy(mnist.sparse_label)
batch_size    = args['batch_size']

new_label = np.copy(sparse_label)
new_label = np.asarray(new_label)

mask_AE  = np.copy(mnist.train_mask)

drops_label_sup=np.ones([batch_size,10])
target_sup=np.ones([batch_size,10])
mask_sup = np.ones([batch_size,10])
mask2_sup = np.ones([batch_size,batch_size,batch_size])
mask_init_sup = np.ones([batch_size,10])
pseudo_labels_sup = np.ones([batch_size,10])

iter = np.arange(int(args['n_epochs']))
scaled_ratio_max = args['ratio_max']
scaled_ratio_max *= 1.0 * args['n_labeled'] / mnist.n_images

print('\n\n[*] Start optimization\n')
with tf.Session() as sess:
    BAE.train()
    sess.run(tf.global_variables_initializer())
    saver = tf.train.Saver()
    tic = time.clock()
    tvars = tf.trainable_variables()
    tvars_vals = sess.run(tvars)

    for epoch in range(int(args['n_epochs'])):
        print("\n\nEpoch {0:03d} / {1:03d}\n".format(epoch, int(args['n_epochs'])))
        rampup_value    = rampup(epoch)
        rampdown_value  = rampdown(epoch)
        learning_rate   = rampup_value * rampdown_value * args['lr_max']
        ratio           = rampup_ratio(epoch) * scaled_ratio_max
        adam_beta1      = rampdown_value * args['adam_beta1'] + (1.0 - rampdown_value) * args['rampdown_beta1']
        max_iter        = np.ceil(rampup_ratio(epoch) * 1)
        hyper           = rampup_value * 1.0

        if epoch < 1:
            for m, batch_idx, images1, images2, target, drops_label, mask in mnist.next_batch(is_training=True):
                ratio = 0.0
                feed_dict = {BAE.data       : images1[mnist.n_labeled:],
                             BAE.labels     : drops_label[mnist.n_labeled:],
                             BAE.targets    : np.zeros_like(target[mnist.n_labeled:]),
                             BAE.mask       : np.zeros_like(mask[mnist.n_labeled:]),
                             BAE.is_train   : True,
                             BAE.is_train2  : False,
                             BAE.ratio      : ratio,
                             BAE.lr_c       : learning_rate,
                             BAE.adam_beta1 : adam_beta1,
                             BAE.pseudo     : np.zeros((args['batch_size'], )),
                             BAE.size       : batch_size,
                             BAE.hyper      : hyper,
                             BAE.is_offset   : True,
                             BAE.use_bae    : True}

                _ = sess.run(BAE.init_op, feed_dict=feed_dict)
                break

        # else:
        total_losses, train_n, sup_losses, graph_losses = 0., 0., 0., 0.
        pos_err, neg_err = 0., 0.
        top1_mc_t        = []
        top1_cnn_t       = []

        use_bae = False
        for m, batch_idx, images1, images2, target, drops_label, mask in mnist.next_batch(is_training=True):

            use_bae = True
            feed_dict = {BAE.data       : images1,
                         BAE.labels     : drops_label,
                         BAE.targets    : target,
                         BAE.mask       : mask,
                         BAE.is_train   : True,
                         BAE.is_train2  : True,
                         BAE.ratio      : ratio,
                         BAE.lr_c       : learning_rate,
                         BAE.lr_b       : 1e-4,
                         BAE.adam_beta1 : adam_beta1,
                         BAE.pseudo     : pseudo_labels_sup,
                         BAE.size       : batch_size,
                         BAE.hyper      : hyper,
                         BAE.is_offset  : False,
                         BAE.use_bae: use_bae}

            _,  loss_BAE,loss_CNN,graph_loss,perturbation,loss_sup,X_filled, check, check2,preds_CNN= sess.run([BAE.Tot_op,BAE.loss_BAE, BAE.loss_CNN,BAE.semi_loss,BAE.entropy_kl,BAE.loss_sup, BAE.MC_Graph,BAE.pos,BAE.neg,BAE.CNN_PRED],
                                          feed_dict=feed_dict)

            arg_pred_mc           = X_filled
            arg_target            = np.argmax(mnist.y_train[batch_idx, :], axis=1)
            top1_mc_t.append(np.mean(arg_target == arg_pred_mc, dtype=np.float32))

            total_losses    += loss_CNN * m
            sup_losses      += loss_sup * m
            graph_losses    += graph_loss * m
            train_n         += m
            pos_err += np.mean(check) * m
            neg_err += np.mean(check2) * m

            arg_pred_cnn = np.argmax(preds_CNN, axis=1)
            top1_cnn_t.append(np.mean(arg_target == arg_pred_cnn, dtype=np.float32))

        toc = time.time()
        print("[*] total loss:%.4e | sup loss: %.4e | perturbation-loss:%.4e | graph-loss:%.4e" % (total_losses / train_n, sup_losses / train_n, perturbation, graph_losses / train_n))
        print("[*] lr: %.7f | beta1: %f | ratio: %f | max_iter: %d" % (learning_rate, adam_beta1, ratio, max_iter))
        print("[*] TRAIN Top-1 CNN Acc: %.4f | BAE Acc: %.4f" % (np.mean(top1_cnn_t), np.mean(top1_mc_t)))
        print("[*] Pos: %f, NEG: %f"%(pos_err, neg_err))

        preds = []
        preds_mc = []
        preds_mc_noise = []
        for m, images1, target in mnist.next_batch(is_training=False):
            feed_dict = {BAE.data       : images1,
                         BAE.labels     : np.zeros([batch_size,10]),
                         BAE.targets    : target,
                         BAE.is_train   : False,
                         BAE.is_train2  : False,
                         BAE.ratio      : ratio,
                         BAE.size       : batch_size,
                         BAE.hyper      : hyper,
                         BAE.is_offset   : True}

            pred, pred_mc = sess.run([BAE.test_preds,
                                      BAE.test_bae],
                                     feed_dict=feed_dict)
            preds.extend(pred)
            preds_mc.extend(pred_mc)

        preds        = np.asarray(preds)
        arg_pred     = np.argmax(preds, axis=1)
        preds_mc     = np.asarray(preds_mc)
        arg_preds_mc = np.argmax(preds_mc, axis=1)

        arg_target  = np.argmax(mnist.y_test[:len(arg_pred)], axis=1)
        top1_acc    = np.mean(arg_target == arg_pred, dtype=np.float32)
        top1_acc_mc = np.mean(arg_target == arg_preds_mc, dtype=np.float32)

        toc = time.clock()
        print("[*] TEST  Top-1 CNN Acc: %.4f | BAE Acc: %.4f | Time: %.4f" % (top1_acc,top1_acc_mc,toc-tic))

        data = [epoch,  np.mean(top1_mc_t), top1_acc]
        csvfile = open('output.csv', 'a')
        csvwriter = csv.writer(csvfile)
        csvwriter.writerow(data)
        csvfile.close()


import unittest

import cv2
import numpy as np

from handwriting import improc

VISUALIZE = False

class TestsImproc(unittest.TestCase):

    """Unit tests for image processing and feature extraction functions."""

    def test_pad_image(self):
        """test function to pad an image"""

        # image is smaller in both dimensions
        image = np.ones((8, 8, 3), dtype=np.uint8) * (0, 0, 255)
        image_padded = improc.pad_image(image, 16, 16)
        self.assertEqual(image_padded.shape, (16, 16, 3))

        if VISUALIZE:
            image_both = np.zeros((16, 32, 3), dtype=np.uint8)
            image_both[0:image.shape[0], 0:image.shape[1], :] = image
            image_both[0:image_padded.shape[0], 16:(image_padded.shape[1] + 16)] = image_padded
            cv2.namedWindow("padding", cv2.WINDOW_NORMAL)
            cv2.imshow("padding", image_both)
            cv2.waitKey()

        # image is larger in both dimensions
        image = np.ones((16, 16, 3), dtype=np.uint8) * (0, 0, 255)
        image_padded = improc.pad_image(image, 8, 8)
        self.assertEqual(image_padded.shape, (8, 8, 3))

        # image is larger in x dimension
        image = np.ones((8, 16, 3), dtype=np.uint8) * (0, 0, 255)
        image_padded = improc.pad_image(image, 8, 8)
        self.assertEqual(image_padded.shape, (8, 8, 3))

        # image is larger in y dimension
        image = np.ones((16, 8, 3), dtype=np.uint8) * (0, 0, 255)
        image_padded = improc.pad_image(image, 8, 8)
        self.assertEqual(image_padded.shape, (8, 8, 3))

        # image is same in both dimensions
        image = np.ones((8, 8, 3), dtype=np.uint8) * (0, 0, 255)
        image_padded = improc.pad_image(image, 8, 8)
        self.assertEqual(image_padded.shape, (8, 8, 3))


from .models import Movie, Genre, Comment
from django.shortcuts import render, get_list_or_404, get_object_or_404
from .serializers.movie_serializers import MovieSerializer, MovieCosSerializer, MovieGenreSerializer, MovieLanguageSerializer
from .serializers.genre_serializers import GenreSerializer
from .serializers.comment_serializers import CommentSerializer
from rest_framework import status
from rest_framework.decorators import api_view, authentication_classes, permission_classes
from rest_framework.response import Response
from rest_framework.permissions import IsAuthenticated
from rest_framework_jwt.authentication import JSONWebTokenAuthentication
from pprint import pprint
from algorithms import cos_algorithms, wr_algorithms

# Create your views here.
@api_view(['GET'])
@authentication_classes([JSONWebTokenAuthentication])
@permission_classes([IsAuthenticated])
def index(request):
    movies = get_list_or_404(Movie)
    serializer = MovieSerializer(movies, many=True)
    return Response(serializer.data)

@api_view(['GET'])
@authentication_classes([JSONWebTokenAuthentication])
@permission_classes([IsAuthenticated])
def recom(request, movie_pk):
    result = []
    cos_similarities = cos_algorithms.cosine_sim_dict[movie_pk]  # 딕셔너리
    for id in cos_similarities:
        if cos_similarities[id] >= 0.12:
            tmp_movie = get_object_or_404(Movie, pk=id)
            tmp_serializer = MovieSerializer(tmp_movie)
            tmp_serializer_dict = dict(tmp_serializer.data)
            tmp_serializer_dict['cos_sim'] = cos_similarities[id]
            result.append(tmp_serializer_dict)
    return Response(result)

@api_view(['GET'])
@authentication_classes([JSONWebTokenAuthentication])
@permission_classes([IsAuthenticated])
def genre_recom(request, genre_pk):
    result = []
    weighted_ratings = wr_algorithms.weighted_ratings
    while len(result) <= 20:
        for id, ratings in weighted_ratings:
            tmp_movie = get_object_or_404(Movie, pk=id)
            tmp_serializer = MovieGenreSerializer(tmp_movie)
            if genre_pk in tmp_serializer.data['genre_ids']:
                result.append(tmp_serializer.data)
    return Response(result)

@api_view(['GET'])
@authentication_classes([JSONWebTokenAuthentication])
@permission_classes([IsAuthenticated])
def movie_detail(request, movie_pk):
    movie = get_object_or_404(Movie, pk=movie_pk)
    serializer = MovieSerializer(movie)
    return Response(serializer.data)

@api_view(['GET'])
@authentication_classes([JSONWebTokenAuthentication])
@permission_classes([IsAuthenticated])
def default_recom(request):
    result = []
    top_twenty = wr_algorithms.weighted_ratings[:20]  # 상위 20개 영화
    for id, rating in top_twenty:
        tmp_movie = get_object_or_404(Movie, pk=id)
        tmp_serializer = MovieSerializer(tmp_movie)
        result.append(tmp_serializer.data)
    return Response(result)

@api_view(['GET'])
@authentication_classes([JSONWebTokenAuthentication])
@permission_classes([IsAuthenticated])
def user_interest(request, language_pk):
    result = []
    weighted_ratings = wr_algorithms.weighted_ratings
    for id, rating in weighted_ratings:
        tmp_movie = get_object_or_404(Movie, pk=id)
        tmp_serializer = MovieLanguageSerializer(tmp_movie)
        if language_pk in tmp_serializer.data['language_ids']:
            result.append(tmp_serializer.data)
    return Response(result)

@api_view(['POST'])
@authentication_classes([JSONWebTokenAuthentication])
@permission_classes([IsAuthenticated])
def create_comment(request, movie_pk):
    user = request.user
    movie = get_object_or_404(Movie, pk=movie_pk)
    serializer = CommentSerializer(data=request.data)
    if serializer.is_valid(raise_exception=True):
        serializer.save(movie=movie, user=user)
        comments = movie.comments.all()
        serializer = CommentSerializer(comments, many=True)
        return Response(serializer.data, status=status.HTTP_201_CREATED)

@api_view(['PUT', 'DELETE'])
@authentication_classes([JSONWebTokenAuthentication])
@permission_classes([IsAuthenticated])
def comment_update_or_delete(request, movie_pk, comment_pk):
    movie = get_object_or_404(Movie, pk=movie_pk)
    comment = get_object_or_404(Comment, pk=comment_pk)

    def update_comment():
        if request.user == comment.user:
            serializer = CommentSerializer(instance=comment, data=request.data)
            if serializer.is_valid(raise_exception=True):
                serializer.save()
                comments = movie.comments.all()
                serializer = CommentSerializer(comments, many=True)
                return Response(serializer.data)

    def delete_comment():
        if request.user == comment.user:
            comment.delete()
            comments = movie.comments.all()
            serializer = CommentSerializer(comments, many=True)
            return Response(serializer.data)

    if request.method == 'PUT':
        return update_comment()
    elif request.method == 'DELETE':
        return delete_comment()


#import io
#from gensim.models.fasttext import FastText as FT_gensim
#from gensim.test.utils import datapath
from gensim.models.keyedvectors import KeyedVectors
import pickle

class Sim_finder:
    def __init__(self, model_file="saved_objects/fasttext/wiki-news-300d-1M.vec", pick_file="saved_objects/fasttext/model.p"):
        try:
            self.model = pickle.load(open(pick_file, "rb"))
            print("loaded fasttext from pickle")
        except:
            print("loading fasttext for the first time")
            self.model = KeyedVectors.load_word2vec_format(model_file)
            print("saving with pickle")
            pickle.dump(self.model, open(pick_file, "wb"))

    def gensim_test():
        model = KeyedVectors.load_word2vec_format("wiki-news-300d-1M.vec")
        print(model.most_similar("nights"))

    def get_close_words(self, positive, negative=None, n=50):
        if type(positive) == str: positive = positive.split()
        return [item[0] for item in self.model.most_similar(positive, negative, topn=n)]


from django.urls import path
from order.views import basket, add_to_basket, remove_from_basket, delivery, take_away, order_status, \
    change_order_status, admin_orders_panel, received_order, expand_quantity, reduce_quantity, \
    admin_order_detail

urlpatterns = [
    path('basket/', basket, name='basket'),
    path('add_to_basket/<int:element_pk>', add_to_basket, name='add_to_basket'),
    path('remove_from_basket/', remove_from_basket, name='remove_from_basket'),
    path('delivery/', delivery, name='delivery'),
    path('take_away/', take_away, name='take_away'),
    path('order_status/', order_status, name='order_status'),
    path('change_order_status/<int:order_pk>', change_order_status, name='change_order_status'),
    path('admin_orders_panel/', admin_orders_panel, name='admin_orders_panel'),
    path('received_order/', received_order, name='received_order'),
    path('expand_quantity/<int:element_pk>', expand_quantity, name='expand_quantity'),
    path('reduce_quantity/<int:element_pk>', reduce_quantity, name='reduce_quantity'),
    path('admin_order_detail/<int:user_pk>', admin_order_detail, name='admin_order_detail'),
]


import os
import sys
import copy
import glob
import shutil
import skimage
import argparse
import scipy.ndimage as ndi
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from toolbox import BasicTools as bt
from toolbox import LIDCTools as lt
from toolbox import Evaluations as eva
from toolbox import CTViewer_Multiax as cvm

def mathcurves_paint():
        x = np.arange(-10.,10.,0.01)
        y = 1 / (1 + np.exp(-x))
        plt.plot(x, y)
        plt.grid(True)
        plt.xlabel("x")
        plt.ylabel("y")
        plt.savefig("sigmoid.png", format='png')
        plt.close()

        y = np.tanh(x)
        plt.plot(x, y)
        plt.grid(True)
        plt.xlabel("x")
        plt.ylabel("y")
        plt.savefig("tanh.png", format='png')
        plt.close()

        y = copy.copy(x)
        y[y<0] = 0
        plt.plot(x, y)
        plt.grid(True)
        plt.xlabel("x")
        plt.ylabel("y")
        plt.savefig("ReLU.png", format='png')
        plt.close()

def samples_replace(source_path="/data/fyl/data_samples/tianchild_cubes_overbound/npy_non_fine", target_path="/data/fyl/data_samples/tianchild_cubes_overbound/npy_non"):
	'''
	filelist = glob.glob(source_path+'/*.npy')
	for file in filelist:
		filename = os.path.basename(file)
		target_file = target_path + '/' + filename
		shutil.copyfile(file, target_file)
		print(target_file)
	'''
	bt.directory_arrange(source_path, target_path, mode='move')

def keras_generator_test():
	from keras.preprocessing.image import ImageDataGenerator
	image = skimage.io.imread("F:/lung_project2/lidc_cubes_64_overbound_roughspacing/LIDC-IDRI-1012_1_2.0_[2.       0.722656 0.722656]_oa_s3.png")
	#image = np.random.rand(50, 50, 3)
	plt.imshow(image)
	plt.savefig('temp.png')
	generator = ImageDataGenerator(shear_range=1., data_format='channels_last', fill_mode='constant')
	image_transformed = generator.random_transform(image)
	plt.imshow(image_transformed)
	plt.show()

def list_compare():
	path1 = "/data/fyl/data_samples/sph_overbound/slicewise"
	path2 = "/data/fyl/data_samples/tianchild_cubes_overbound2/npy"
	samples1 = os.listdir(path1)
	samples2 = os.listdir(path2)
	patients1 = []
	patients2 = []
	for sample in samples1:
		info = '_'.join(os.path.basename(sample).split('_')[:2])
		#info = os.path.basename(sample).split('_')[0]
		if info not in patients1:
			patients1.append(info)
	for sample in samples2:
		#info = '_'.join(os.path.basename(sample).split('_')[:2])
		info = os.path.basename(sample).split('_')[0]
		if info not in patients2:
			patients2.append(info)
	'''
	samples1 = os.listdir("/data/fyl/data_samples/sph2015_overbound/npy")
	samples1.extend(os.listdir("/data/fyl/data_samples/sph2015_overbound/npy_non"))
	statsdict1 = {}
	for sample in samples1:
		patient = sample.split('_')[0]
		if patient in statsdict1.keys():
			statsdict1[patient] += 1
		else:
			statsdict1[patient] = 1
	stats1 = []
	for statitem in statsdict1.items():
		stats1.append(statitem[0]+':'+str(statitem[1]))
	stats2 = bt.filelist_load("/home/fyl/programs/lung_project/data_samples/sph2015_overbound/nodstats.log")[:-1]
	'''
	set1 = set(patients1)
	set2 = set(patients2)
	set_diff1 = set2.difference(set1)
	set_diff2 = set1.difference(set2)
	print(set_diff1)
	print(set_diff2)

def detection_evaluation():
	annotation_file = "/data/fyl/datasets/LUNA16/csvfiles/annotations_corrected.csv"
	#result_file = "pytorch_project/experiments_dt/evaluations_test2/result.csv"
	evaluation_path = 'pytorch_project/experiments_dt/evaluations_sliccand_densecropnet_e11_fold4'
	#eva.csv_evaluation(annotation_file, result_file, evaluation_path, exclude_file=None)
	#results = np.load('pytorch_project/experiments_dt/evaluations_sliccand_densecropnet_e11_fold4/result.npy')
	results_csv = pd.read_csv('pytorch_project/experiments_dt/evaluations_sliccand_densecropnet_e11_fold4/result.csv')
	results = []
	for r in range(len(results_csv["seriesuid"].values)):
		uid = results_csv["seriesuid"].values[r]
		coordX = results_csv["coordX"].values[r]
		coordY = results_csv["coordY"].values[r]
		coordZ = results_csv["coordZ"].values[r]
		prob = results_csv["probability"].values[r]
		results.append([uid, coordX, coordY, coordZ, prob])
	#result_order = results[:,-1].argsort()[::-1]
	#for oi in result_order:
	#	print(results[oi])
	assessment = eva.detection_assessment(results, annotation_file, label=1)
	eva.evaluation_vision([], assessment['num_scans'], assessment['FROC'][0], assessment['FROC'][1], assessment['CPM'], assessment['detection_cites'], output_path=evaluation_path)

def filename_arrange(source_path="sph_cubes_64_overbound/npy", target_path="lung adenocarcinoma samples of the private dataset"):
	filenames = os.listdir(source_path)
	ids = []
	for filename in filenames:
		fnsplit = os.path.splitext(filename)[0].split('_')
		if len(fnsplit)>1:
			if (fnsplit[0], fnsplit[1]) not in ids: ids.append((fnsplit[0], fnsplit[1]))
			idx = ids.index((fnsplit[0], fnsplit[1])) + 1
			label = fnsplit[2]
			#pidx = fnsplit[-1][1:]
			#newname = "sample%d_%s_patch%s.png" %(idx, label, pidx)
			newname = "sample%d_%s.npy" %(idx, label)
			newpath = target_path+'/'+label
			if not os.path.exists(newpath):
				os.makedirs(newpath)
			shutil.copy(source_path+'/'+filename, newpath+'/'+newname)

def featuremap_visualize(files, names = [], num_channels = 20):
	fig = plt.figure()
	#fig = plt.figure(figsize=(12, 3.5))
	#plt.title('channels')
	#fig, ax = plt.subplots(len(files), 1)
	fms = []
	axs = []
	for f in range(len(files)):
		if isinstance(files[f], str):
			featuremap = np.load(files[f])
			ax = fig.add_subplot(len(files), 1, f+1)
			fm = ax.imshow(featuremap[0], vmin=0, vmax=1)
			ax.set_xticks(np.append(np.arange(0, featuremap.shape[2], featuremap.shape[2]//num_channels), featuremap.shape[2]-1))
			ax.set_xticklabels(np.arange(0, num_channels+1))
			ax.set_yticks([])
			#ax.set_yticks([0, featuremap.shape[1]-1])
			#ax.set_yticklabels([0, 1])
			#ax.set_title('channels')
			if f==0:
				ax.set_xlabel('channels')
				ax.xaxis.set_label_position('top')
			ax.set_ylabel('layer %d' %(f + 1))
			axs.append(ax)
			fms.append(fm)
		else:
			aaxs = []
			file_nums = [len(file_approach) for file_approach in files]
			for l in range(len(files[f])):
				featuremap = np.load(files[f][l])
				ax = fig.add_subplot(sum(file_nums), 1, sum(file_nums[:f])+l+1)
				fm = ax.imshow(featuremap[0], vmin=0, vmax=1)
				ax.set_xticks(np.append(np.arange(0, featuremap.shape[2], featuremap.shape[2]//num_channels), featuremap.shape[2]-1))
				ax.set_xticklabels(np.arange(0, num_channels+1))
				ax.set_yticks([])
				ax.set_ylabel('layer %d'%(l*3+1), rotation=0)
				aaxs.append(ax)
				fms.append(fm)
			aaxs[-1].set_xlabel(names[f])
			#aaxs[0].set_title('channels')
			axs.extend(aaxs)
	#plt.subplots_adjust(left=-1, hspace=5)
	fig.colorbar(fms[0], ax=axs)
	plt.show()
	plt.close()

featuremap_visualize(files=[['fmn1l1.npy', 'fmn1l4.npy'], ['fmtr1l1.npy', 'fmtr1l4.npy'], ['fmt1l1.npy', 'fmt1l4.npy'], ['fms1l1.npy', 'fms1l4.npy']], names=['DNC learning simply on LIDC-IDRI', 'transfer learning', 'multitask learning', 'bias-undoing learning'])
'''
parser = argparse.ArgumentParser(description='Process some integers.')
parser.add_argument('integers', metavar='N', type=int, nargs='+',
                    help='an integer for the accumulator')
parser.add_argument('--sum', dest='accumulate', action='store_const',
		    const=sum, default=max, help='sum the integers (default: find the max)')
args = parser.parse_args()
print(args.accumulate(args.integers))
'''
#detection_evaluation()


import tempfile
from collections import UserList, deque
from collections.abc import Mapping
from contextlib import contextmanager
from dataclasses import dataclass, field, InitVar
from itertools import starmap
from time import time
from typing import Sequence

# -------------- External Library -------------- #

from inflection import underscore
from aenum import Enum, AutoValue
from path import Path

# -------------- Hexfarm  Library -------------- #

from ..shell import decoded, Command, me
from ..util import classproperty, map_value_or, value_or

__all__ = (
    "CondorCommand",
    "CONDOR_COMMANDS",
    "current_jobs",
    "user_jobs",
    "Universe",
    "Notification",
    "FileTransferMode",
    "TransferOutputMode",
    "This",
    "condor_format_mapping",
    "condor_format_sequence",
    "extract_job_ids",
    "CONDOR_SUBMIT_COMMANDS",
    "JobConfig",
    "minimal_config",
    "submit_config",
    "Job",
    "job_dict",
    "JobCompletedException",
    "JobMap",
    "ConfigUnitBase",
    "ProcessUnit",
    "ClusterUnit",
    "MultiClusterUnit",
    "ConfigRunner",
    "JobManager",
)

class CondorCommand(Command, prefix="condor_"):
    """Condor Command Object."""

CONDOR_COMMANDS = (
    "advertise",
    "c-gahp",
    "c-gahp_worker_thread",
    "check_userlogs",
    "checkpoint",
    "ckpt_server",
    "cod",
    "collector",
    "compile",
    "config_val",
    "configure",
    "continue",
    "credd",
    "dagman",
    "drain",
    "fetchlog",
    "findhost",
    "ft-gahp",
    "gather_info",
    "gridmanager",
    "gridshell",
    "had",
    "history",
    "hold",
    "init",
    "install",
    "kbdd",
    "master",
    "master_s",
    "negotiator",
    "off",
    "on",
    "ping",
    "power",
    "preen",
    "prio",
    "procd",
    "q",
    "qedit",
    "qsub",
    "reconfig",
    "release",
    "replication",
    "reschedule",
    "restart",
    "rm",
    "root_switchboard",
    "router_history",
    "router_q",
    "router_rm",
    "run",
    "schedd",
    "set_shutdown",
    "shadow",
    "shadow.std",
    "shadow_s",
    "ssh_to_job",
    "startd",
    "starter",
    "starter.std",
    "stats",
    "status",
    "store_cred",
    "submit",
    "submit_dag",
    "suspend",
    "tail",
    "test_match",
    "transfer_data",
    "transferd",
    "updates_stats",
    "userlog",
    "userlog_job_counter",
    "userprio",
    "vacate",
    "vacate_job",
    "version",
    "vm-gahp",
    "vm-gahp-vmware",
    "vm_vmware",
    "vm_vmware.pl",
    "wait",
    "who",
)

def add_condor_commands(commands, command_dict):
    """Build Condor Commands for Globals."""
    name_list = []
    for name in commands:
        setname = (CondorCommand.prefix + name).replace(".", "").replace("-", "_")
        name_list.append(setname)
        command_dict[setname] = CondorCommand(name)
    return tuple(name_list)

__all__ += add_condor_commands(CONDOR_COMMANDS, globals())

def current_jobs(*usernames):
    """List the Current Jobs by Users."""
    query = condor_q(" ".join(usernames)) if usernames else condor_q()
    text = decoded(query).strip().split("\n")
    user_dict = dict()
    for job_id, name, *_ in list(map(lambda l: l.strip().split(), text))[2:-2]:
        if name not in user_dict:
            user_dict[name] = []
        user_dict[name].append(job_id)
    return user_dict

def user_jobs(username=me()):
    """Get User's Jobs."""
    return current_jobs(username)[username]

class NameEnum(Enum, settings=AutoValue):
    """Named Enum Objects."""

    def __str__(self):
        """Get Name as String."""
        return self.name

class Universe(NameEnum):
    """Condor Universe Enum."""

    Vanilla, Standard, Scheduler, Local, Grid, Java, VM, Parallel, Docker

@dataclass
class Notification:
    """Notification Structure."""

    class Status(NameEnum):
        """Notification Status."""

        Always, Complete, Error, Never

    email: str
    status: Status
    attributes: Sequence[str] = field(default_factory=list)

    def __str__(self):
        """Get String Representation of Notification."""
        attributes = f'email_attributes={",".join(self.attributes)}'
        return "\n".join(
            (
                f"notification={self.status}",
                f"notify-user={self.email}",
                f'{attributes if self.attributes else ""}',
            )
        )

class FileTransferMode(NameEnum):
    """File Transfer Mode."""

    Yes, No, IfNeeded

class TransferOutputMode(NameEnum):
    """Transfer Output Mode."""

    OnExit, OnExitOrEvict

class This(NameEnum):
    """Universal Condor Object Variables."""

    ClusterId, Cluster, ProcId, Process, Node, Step, ItemIndex, Row, Item

    def __str__(self):
        """Get String Form of Variable."""
        return f"$({super().__str__()})"

def kv_string(key, value, *, connector="="):
    """Make Key-Value String."""
    return f"{key}{connector}{value}"

def condor_format_mapping(mapping):
    """Format a Mapping According to Condor Conventions."""
    return ";".join(starmap(kv_string, mapping.items()))

def condor_format_sequence(sequence):
    """Format a Sequence According to Condor Conventions."""
    return " ".join(map(str, sequence))

def open_key_value_pairs(self, *pairs, connector="=", ignore_unpacking_error=True):
    """Open Pair and Check for Keys and Values."""
    if not pairs:
        return None, None
    if len(pairs) == 1:
        try:
            key, value = tuple(map(lambda o: o.strip(), pairs[0].split(connector)))
            return ((key, value),)
        except ValueError as value_error:
            if ignore_unpacking_error:
                return ((None, None),)
            raise value_error
    else:
        # TODO: replace recursive implementation
        return sum(tuple(open_key_value_pairs(line) for line in pairs), ())

def extract_job_ids(condor_submit_text):
    """Extract Job Ids from Condor Submit Text."""
    _, _, count, *_, cluster = condor_submit_text.strip().split()
    for process in map(str, range(int(count))):
        yield cluster + process

class WriteModeMixin:
    """
    Writing Mode Mixin Class.

    """

    def __init_subclass__(
        cls, write_mode_keywords, write_mode_invalid_keywords=None, **kwargs
    ):
        """Initialize Write-Mode Keywords for Class."""
        super().__init_subclass__(**kwargs)
        cls._write_mode_keywords = set(write_mode_keywords)
        cls._write_mode_invalid_keywords = map_value_or(
            set, write_mode_invalid_keywords, set()
        )

    def _write_mode_insert(self, name, value):
        """Insert Key-Value Pair into Object."""
        return NotImplemented

    def _write_mode_invalid_keyword(self, name, value):
        """Check if Write-Mode Key is valid."""
        raise ValueError(f"Key: {name} is an invalid Keyword.")

    @property
    @contextmanager
    def write_mode(self):
        """Writing Mode Context Manager."""
        try:
            self._write_mode = True
            yield self
        finally:
            self._write_mode = False

    def __setattr__(self, name, value):
        """Set attributes."""
        if self._write_mode:
            if name in type(self)._write_mode_invalid_keywords:
                self._write_mode_invalid_keyword(name, value)
            elif name in type(self)._write_mode_keywords:
                self._write_mode_insert(name, value)
        else:
            super().__setattr__(name, value)

CONDOR_SUBMIT_COMMANDS = {
    "accounting_group",
    "accounting_group_user",
    "allow_startup_script",
    "append_files",
    "arguments",
    "azure_admin_key",
    "azure_admin_username",
    "azure_auth_file",
    "azure_image",
    "azure_location",
    "azure_size",
    "batch_queue",
    "boinc_authenticator_file",
    "buffer_block_size",
    "buffer_files",
    "buffer_size",
    "compress_files",
    "concurrency_limits",
    "concurrency_limits_expr",
    "copy_to_spool",
    "coresize",
    "cream_attributes",
    "cron_day_of_month",
    "cron_day_of_week",
    "cron_hour",
    "cron_minute",
    "cron_month",
    "cron_prep_time",
    "cron_window",
    "dagman_log",
    "deferral_prep_time",
    "deferral_time",
    "deferral_window",
    "delegate_job_GSI_credentials_lifetime",
    "description",
    "docker_image",
    "dont_encrypt_input_files",
    "dont_encrypt_output_files",
    "ec2_access_key_id",
    "ec2_ami_id",
    "ec2_availability_zone",
    "ec2_block_device_mapping",
    "ec2_ebs_volumes",
    "ec2_elastic_ip",
    "ec2_iam_profile_arn",
    "ec2_iam_profile_name",
    "ec2_instance_type",
    "ec2_keypair",
    "ec2_keypair_file",
    "ec2_parameter_",
    "ec2_parameter_names",
    "ec2_secret_access_key",
    "ec2_security_groups",
    "ec2_security_ids",
    "ec2_spot_price",
    "ec2_tag_",
    "ec2_tag_names",
    "ec2_user_data",
    "ec2_user_data_file",
    "ec2_vpc_ip",
    "ec2_vpc_subnet",
    "email_attributes",
    "encrypt_execute_directory",
    "encrypt_input_files",
    "encrypt_output_files",
    "environment",
    "error",
    "executable",
    "fetch_files",
    "file_remaps",
    "gce_auth_file",
    "gce_image",
    "gce_json_file",
    "gce_machine_type",
    "gce_metadata",
    "gce_metadata_file",
    "gce_preemptible",
    "getenv",
    "globus_rematch",
    "globus_resubmit",
    "globus_rsl",
    "grid_resource",
    "hold",
    "hold_kill_sig",
    "image_size",
    "initialdir",
    "input",
    "jar_files",
    "java_vm_args",
    "job_ad_information_attrs",
    "job_batch_name",
    "job_lease_duration",
    "job_machine_attrs",
    "job_max_vacate_time",
    "keep_claim_idle",
    "keystore_alias",
    "keystore_file",
    "keystore_passphrase_file",
    "kill_sig",
    "kill_sig_timeout",
    "leave_in_queue",
    "load_profile",
    "local_files",
    "log",
    "log_xml",
    "machine_count",
    "match_list_length",
    "max_job_retirement_time",
    "max_retries",
    "max_transfer_input_mb",
    "max_transfer_output_mb",
    "my_proxy_credential_name",
    "my_proxy_host",
    "my_proxy_new_proxy_lifetime",
    "my_proxy_password",
    "my_proxy_refresh_threshold",
    "my_proxy_server_dn",
    "next_job_start_delay",
    "nice_user",
    "noop_job",
    "noop_job_exit_code",
    "noop_job_exit_signal",
    "nordugrid_rsl",
    "notification",
    "on_exit_hold",
    "on_exit_hold_reason",
    "on_exit_hold_subcode",
    "on_exit_remove",
    "output",
    "output_destination",
    "periodic_hold",
    "periodic_hold_reason",
    "periodic_hold_subcode",
    "periodic_release",
    "periodic_remove",
    "priority",
    "queue",
    "rank",
    "remote_initialdir",
    "remove_kill_sig",
    "rendezvousdir",
    "request_",
    "request_cpus",
    "request_disk",
    "request_memory",
    "requirements",
    "retry_until",
    "run_as_owner",
    "should_transfer_files",
    "skip_filechecks",
    "stack_size",
    "stream_error",
    "stream_input",
    "stream_output",
    "submit_event_notes",
    "success_exit_code",
    "transfer_error",
    "transfer_executable",
    "transfer_input",
    "transfer_input_files",
    "transfer_output",
    "transfer_output_files",
    "transfer_output_remaps",
    "universe",
    "use_x509userproxy",
    "vm_checkpoint",
    "vm_disk",
    "vm_macaddr",
    "vm_memory",
    "vm_networking",
    "vm_networking_type",
    "vm_no_output_vm",
    "vm_type",
    "vmware_dir",
    "vmware_should_transfer_files",
    "vmware_snapshot_disk",
    "want_graceful_removal",
    "want_name_tag",
    "want_remote_io",
    "when_to_transfer_output",
    "x509userproxy",
    "xen_initrd",
    "xen_kernel",
    "xen_kernel_params",
    "xen_root",
}

class JobConfig(UserList, WriteModeMixin, write_mode_keywords=CONDOR_SUBMIT_COMMANDS):
    """
    Job Configuration Base.

    """

    @classproperty
    def command_names(cls):
        """Get Possible Command Names for Condor Config File."""
        return cls._write_mode_keywords

    @classmethod
    def from_file(
        cls, path, opener=open, *args, clean_input=str.strip, keep_path=True, **kwargs
    ):
        """Load Job Config from File."""
        with opener(path, *args, **kwargs) as file:
            return cls(
                *(clean_input(line) for line in file), path=path if keep_path else None
            )

    def __init_subclass__(cls, special_key_map=None, **kwargs):
        """Initialize Subclasses with Special Key Maps."""
        super().__init_subclass__(**kwargs)
        if special_key_map is not None:
            cls.special_key_map = special_key_map

    def __init__(self, *lines, path=None):
        """Initialize Config."""
        self.__dict__["_write_mode"] = False
        super().__init__(*lines)
        self._path = path

    def _build_path_save_mechanism(self):
        """Build Path Save Mechanism."""
        if hasattr(self, "save"):
            raise AttributeError(
                "JobConfig has no attribute _build_path_save_mechanism."
            )

        def _save(s):
            Path(s.path).write_text(s.as_text)

        def _save_as(s, new_path):
            s._path = new_path
            return s.save()

        self.save = _save
        self.save.__doc__ = """Save Config to File."""
        self.save_as = _save_as
        self.save_as.__doc__ = """Save Config at New Path."""

    @property
    def path(self):
        """Get Path of Config if Exists."""
        if self._path is None:
            raise AttributeError("JobConfig has no attribute path.")
        try:
            self._build_path_save_mechanism()
        except AttributeError:
            pass
        return self._path

    @path.setter
    def path(self, value):
        """Set Path of Config if Exists."""
        self._path = value
        try:
            self._build_path_save_mechanism()
        except AttributeError:
            pass

    @property
    def lines(self):
        """Alias to Inner Structure."""
        return self.data

    @property
    def lines(self, new_lines):
        """Set Inner Structure."""
        self.data = new_lines

    def to_text(self, *, with_newlines=True):
        """Return Config as Multiline Text."""
        return ("\n" if with_newlines else "").join(self)

    @property
    def as_text(self):
        """Return Config as Multiline Text."""
        return self.to_text()

    def __repr__(self):
        """Representation of Config File."""
        data_string = (
            f'[{str(self.data[0:6])[1:-1]}{", ...]" if len(self) > 6 else "]"}'
        )
        return f"{type(self).__name__}({data_string})"

    def __str__(self):
        """Get Config File as a String."""
        return self.as_text

    def append(self, value, *, skip_special_key_search=False):
        """Append to Config."""
        if not skip_special_key_search and hasattr(type(self), "special_key_map"):
            for key, internal_value in open_key_value_pairs(*value.strip().split("\n")):
                try:
                    special_name, factory = type(self).special_key_map[key]
                    setattr(self, special_name, factory(internal_value))
                except KeyError:
                    pass
        super().append(str(value))

    def insert(self, index, value):
        """Insert Value at Index."""
        return NotImplemented

    def append_key_value_pair(self, key, value, *args, **kwargs):
        """Append Key Value Pair to Config."""
        return self.append(kv_string(key, value), *args, **kwargs)

    def _write_mode_insert(self, name, value):
        """Insert Key-Value Pair into Object."""
        return self.append_key_value_pair(name, value)

    def extend(self, other, *args, **kwargs):
        """Extend Config by another Config."""
        for obj in other:
            self.append(str(obj), *args, **kwargs)

    def add_pairs(self, pairs, *args, **kwargs):
        """Append Key Value Pairs to Config."""
        self.extend(starmap(kv_string, pairs.items()), *args, **kwargs)

    def __add__(self, other):
        """Create Sum of Config."""
        if not isinstance(other, type(self)):
            return super().__add__(map(str, other))
        return super().__add__(other)

    def __iadd__(self, other):
        """Add to Config In Place."""
        self.extend(list(other))
        return self

    def with_arguments(self, *args):
        """Add Arguments to Config."""
        self.append_key_value_pair("arguments", condor_format_sequence(args))

    def append_comments(self, *comments):
        """Add Comments to Config."""
        if not comments:
            return
        if len(comments) == 1:
            comments = comments[0].split("\n")
        self.extend(map(lambda c: f"# {c}", comments))

    def append_email_notification(self, *notification):
        """Add Email Notification to Config."""
        if len(notification) == 1 and isinstance(notification[0], Notification):
            notification = str(notification)
        elif len(notification) == 2:
            notification = str(Notification(*notification))
        else:
            raise TypeError("Not a Notification Type.")
        self.extend(notification.split("\n"))

    def file_remapping(self, key="file_remaps", **mapping):
        """Add File Remapping to Config."""
        self.append_key_value_pair(
            str(key), condor_format_mapping(mapping), skip_special_key_search=True
        )

    def transfer_output_remapping(self, **mapping):
        """Transfer File Output Remapping."""
        self.file_remapping(key="transfer_output_remaps", **mapping)

    def request_(self, name, value):
        """Special Request Tag."""
        self.append_key_value_pair(f"request_{name}", value)

    def queue(self, *args):
        """Add Queue to Config."""
        self.append(f'queue{(" " + condor_format_sequence(args)) if args else ""}')

def minimal_config(
    name,
    executable,
    directory,
    *,
    config_ext="cfg",
    log_ext="log",
    out_ext="out",
    error_ext="err",
    use_absolute_paths=True,
    keep_env=False,
):
    """Construct Minimal Job Configuration."""
    directory = Path(directory)
    if use_absolute_paths:
        build_path = lambda ext: (directory / f"{name}.{ext}").abspath()
    else:
        build_path = lambda ext: directory / f"{name}.{ext}"
    config = JobConfig(path=build_path(config_ext))
    config.name = property(lambda s: name)
    with config.write_mode as config:
        config.executable = Path(executable).abspath()
        config.log = build_path(log_ext)
        config.getenv = keep_env
        config.initialdir = directory.abspath() if use_absolute_paths else directory
        config.output = build_path(out_ext)
        config.error = build_path(error_ext)
    return config

def submit_config(config, path=None, logfile=None, *args, **kwargs):
    """Submit Configuration File."""
    # FIXME: (DRY) generalize path making
    if path is None:
        with tempfile.TemporaryDirectory() as temp_dir:
            temp_path = Path(temp_dir) / "config.txt"
            temp_path.touch()
            temp_path.write_text(config.as_text)
            submit_output = decoded(condor_submit(temp_path, *args, **kwargs))
    else:
        if not path.exists():
            path.parent.makedirs_p()
            path.touch()
            path.write_text(config.as_text)
        submit_output = decoded(condor_submit(path, *args, **kwargs))
    return tuple(
        Job(config, job_id, logfile=logfile)
        for job_id in extract_job_ids(submit_output)
    )

class Job:
    """
    Job Object.

    """

    def __init__(self, config, job_id, submitter=me(), logfile=None):
        """Initialize Job Object."""
        self._config = config
        self._job_id = job_id
        self._submitter = submitter
        self._logfile = logfile
        if logfile is None:
            try:
                self._logfile = self.config.logfile
            except AttributeError:
                pass

    @property
    def config(self):
        """Get Configuration for Job."""
        return self._config

    @property
    def job_id(self):
        """Get JobID of Job."""
        return self._job_id

    @property
    def logfile(self):
        """Get Path of Log File for this Job."""
        return self._logfile

    @property
    def submitter(self):
        """Job Submitter."""
        return self._submitter

    @property
    def is_empty_job(self):
        """Check if Job is Empty."""
        return self._job_id is None or self._config is None

    def remove(self, *args, **kwargs):
        """Remove Job."""
        result = condor_rm(self.job_id, *args, **kwargs)
        self._job_id = None
        return result

    def hold(self, *args, **kwargs):
        """Hold Job."""
        return condor_hold(self.job_id, *args, **kwargs)

    def release(self, *args, **kwargs):
        """Release Job."""
        return condor_release(self.job_id, *args, **kwargs)

    def check(self, wait_timeout=None):
        """Check if Job Has Completed."""
        args = ("-wait", str(wait_timeout)) if wait_timeout else ()
        if self.logfile:
            return not bool(condor_wait(Path(self.logfile).abspath(), *args).returncode)
        else:
            return self.job_id not in user_jobs(self.submitter)

    @property
    def has_completed(self):
        """Check if Job Has Completed."""
        return self.check(wait_timeout=1)

    def on_completion(self, f, *args, wait_timeout=None, **kwargs):
        """Run On Completion of Job."""
        if self.check(wait_timeout=wait_timeout):
            return f(self, *args, **kwargs)
        return False

    def __repr__(self):
        """Representation of Job."""
        return str(self)

    def __str__(self):
        """String conversion of Job."""
        return f"Job({self.job_id})"

def job_dict(*jobs):
    """Make Job Dictionary."""
    return {job.job_id: job for job in jobs}

class JobCompletedException(KeyError):
    """Job Completed Exception."""

    def __init__(self, job):
        """Initialize Exception."""
        super().__init__(f"Job {job.job_id} has already completed.")
        self._job = job

    @property
    def job(self):
        """Get Job which had Completed."""
        return self._job

class JobMap(Mapping):
    """
    Job Mapping Object.

    """

    def __init__(self, *jobs, remove_completed_jobs=False, remove_when_clearing=True):
        """Initialize Job Mapping."""
        self._jobs = job_dict(*jobs)
        self.remove_completed_jobs = remove_completed_jobs
        self.remove_when_clearing = remove_when_clearing

    @property
    def job_ids(self):
        """Return Job Ids."""
        return self.keys()

    @property
    def jobs(self):
        """Return Jobs."""
        return self.values()

    def __getitem__(self, key):
        """Get the Job at given Id."""
        job = self._jobs[key]
        if self.remove_completed_jobs and job.has_completed:
            del self._jobs[key]
            raise JobCompletedException(job)
        return job

    def pop_completed(self):
        """Pop off Jobs that have Completed."""
        out = []
        for job_id, job in tuple(self._jobs.items()):
            if job.has_completed:
                out.append(job)
                del self._jobs[job_id]
        return tuple(out)

    def __iter__(self):
        """Return Iterator over Jobs."""
        if self.remove_completed_jobs:
            self.pop_completed()
        return iter(self._jobs)

    def __len__(self):
        """Return Length of Job List."""
        if self.remove_completed_jobs:
            self.pop_completed()
        return len(self._jobs)

    def clear(self):
        """Clear All Jobs."""
        if self.remove_when_clearing:
            for job_id in self:
                self.remove_job(job_id)
        else:
            self._jobs.clear()

    def update(self, other):
        """Extend Job Map."""
        self._jobs.update(other)

    def append(self, *jobs):
        """Append Jobs to JobMap."""
        self.update(job_dict(*jobs))

    def pop(self, key, *default):
        """Pop Job Out of Map."""
        if len(default) not in (0, 1):
            raise TypeError("Only 1 Default Argument Allowed.")
        try:
            if self.remove_when_clearing:
                return self.remove_job(key)[0]
            else:
                return self._jobs.pop(key)
        except KeyError:
            if len(default) == 1:
                return default
            raise KeyError("Job Not Found.")

    def remove_job(self, job_id, *args, **kwargs):
        """Remove Job."""
        job = self[job_id]
        result = job.remove(*args, **kwargs)
        del self._jobs[job_id]
        return job, result

    def hold_job(self, job_id, *args, **kwargs):
        """Hold Job."""
        return self[job_id].hold(*args, **kwargs)

    def release_job(self, job_id, *args, **kwargs):
        """Release Job."""
        return self[job_id].release(*args, **kwargs)

    def __repr__(self):
        """Representation of JobMap."""
        return repr(self._jobs)

    def __str__(self):
        """String Conversion of JobMap."""
        return str(self._jobs)

class ConfigUnitBase(WriteModeMixin, write_mode_keywords=set()):
    """Configuration Unit Base Structure."""

    def __init_subclass__(
        cls,
        class_ad_name,
        class_ad_id_name,
        fixed_keys,
        make_prefix_config=True,
        **kwargs,
    ):
        """Initialize Configuration Subclasses."""
        super().__init_subclass__(**kwargs)
        setattr(
            cls,
            underscore(class_ad_name),
            classproperty(lambda c: f"$({class_ad_name})"),
        )
        setattr(
            cls,
            underscore(class_ad_id_name),
            classproperty(lambda c: f"$({class_ad_id_name})"),
        )
        for name in fixed_keys:
            fget = (
                lambda s, n=name: kv_string(n, getattr(s, n)) if getattr(s, n) else ""
            )
            setattr(cls, f"{underscore(name)}_kv_str", property(fget))
        if make_prefix_config:

            def _prefix_config(s):
                return JobConfig(
                    getattr(s, f"{underscore(name)}_kv_str") for name in fixed_keys
                )

            setattr(cls, "prefix_config", property(_prefix_config))

    def __init__(self, *args, config_path=None, **kwargs):
        """Initialize Configuration Unit."""
        super().__init__(*args, **kwargs)
        self.path = config_path

    def __str__(self):
        """Get String Representation of ConfigUnit."""
        return str(self.config)

class ProcessUnit(
    ConfigUnitBase,
    class_ad_name="Process",
    class_ad_id_name="ProcId",
    fixed_keys=("initialdir", "output", "error"),
    write_mode_keywords=CONDOR_SUBMIT_COMMANDS,
    write_mode_invalid_keywords=("executable", "log"),
):
    """
    Process Configuration Unit.

    """

    def __init__(self, executable, initialdir=None, output=None, error=None, **kwargs):
        """Initialize Process Configuration Unit."""
        super().__init__(**kwargs)
        self.executable = executable
        self.initialdir = map_value_or(Path, initialdir, None)
        self.output = map_value_or(Path, output, None)
        self.error = map_value_or(Path, error, None)
        self._config = JobConfig()

    def _write_mode_insert(self, name, value):
        """Insert Key-Value Pair into Internal Config."""
        self._config._write_mode_insert(name, value)

    def _write_mode_invalid_keyword(self, name, value):
        """Catch Invalid Keywords for Write-Mode."""
        if name == "executable":
            raise TypeError("Cannot modify executable in ProcessUnit Context.")
        if name == "log":
            raise TypeError("Cannot modify log in ProcessUnit Context.")

    @property
    def config(self):
        """Full Configuration."""
        return self.prefix_config + self._config

    def queue(self, n=1, *args, **kwargs):
        """Add Queue Command."""
        self._config.queue(n=n, *args, **kwargs)

    def queue_in(self, n=1, *args, **kwargs):
        """"""
        return NotImplemented

    def queue_matching(self, n=1, *args, **kwargs):
        """"""
        return NotImplemented

    def queue_from(self, n=1, *args, **kwargs):
        """"""
        return NotImplemented

class ClusterUnit(
    ConfigUnitBase,
    class_ad_name="Cluster",
    class_ad_id_name="ClusterId",
    fixed_keys=("executable", "log"),
    write_mode_keywords=CONDOR_SUBMIT_COMMANDS,
):
    """
    Cluster Configuration Unit.

    """

    def __init__(self, executable, log=None, *process_units, **kwargs):
        """Initialize Cluster Configuration Unit."""
        super().__init__(**kwargs)
        self.executable = executable
        self.log = map_value_or(Path, log, None)
        self.process_units = process_units

    def append_process(self, process_unit):
        """Append Process to Cluster Unit."""
        self.process_units.append(process_unit)

    def append_process_units(self, *process_units):
        """Extend Process Units."""
        self.process_units.extend(process_units)

    @property
    def config(self):
        """Full Configuration."""
        return sum(self.process_units, self.prefix_config)

    def save(self, path=None):
        """Save Config To Path."""
        path = value_or(path, self.path)
        if not path:
            raise TypeError(
                "Path argument must be a valid file path if "
                "stored config has no associated path."
            )
        Path(path).write_text(self.config.as_text)

    def submit(self, *args, use_temporary_file=False, **kwargs):
        """Submit Config As a Condor Job."""
        path = None if use_temporary_file else Path(self.path)
        if path and not path.exists():
            raise FileNotFoundError(f"Path {path} cannot be found.")
        return submit_config(self.config, path=path, logfile=self.log, *args, **kwargs)

class MultiClusterUnit:
    """
    Multi-Cluster Configuration Unit.

    """

    def __init__(self, prefix_config, *cluster_units):
        """Initialize Multi-Cluster Unit."""
        self.prefix_config = prefix_config
        self.cluster_units = cluster_units

    def append_cluster(self, cluster_unit):
        """Append Cluster to Units."""
        self.cluster_units.append(cluster_unit)

    def append_custer_units(self, *cluster_units):
        """Extend Cluster Units."""
        self.cluster_units.extend(cluster_units)

    @property
    def total_config(self):
        """Total Configuration."""
        return sum(self.cluster_units, self.prefix_config)

    def submit(self, *args, use_temporary_file=False, **kwargs):
        """Submit Config As a Condor Job."""
        # FIXME: figure out how to parse a multicluster unit condor_submit output so this can be
        #       joined into one file
        return tuple(
            cluster.submit(use_temporary_file=use_temporary_file, *args, **kwargs)
            for cluster in self.cluster_units
        )

@dataclass
class ConfigRunner:
    """
    Configuration Runner.

    """

    config: JobConfig
    path: InitVar[Path]
    logfile: Path
    jobmap: JobMap

    def __post_init__(self, path):
        """Post-Initialize Config Runner."""
        if Path(path).abspath() != Path(self.config.path).abspath():
            self.config.save_as(path)

    @property
    def config_path(self):
        """Get Configuration Path."""
        return self.config.path

    @property
    def running_job_count(self):
        """Get Current Running Job Count."""
        return len(self.jobmap)

    def submit(self, *args, **kwargs):
        """Submit Jobs."""
        jobs = submit_config(
            self.config, path=self.config_path, logfile=self.logfile, *args, **kwargs
        )
        self.jobmap.append(*jobs)
        return jobs

    def submit_while(self, predicate, *args, wait=None, **kwargs):
        """Submit Jobs while predicate holds."""
        while predicate(self):
            self.submit(*args, **kwargs)
            if wait:
                time.sleep(wait)

class JobManager:
    """
    Job Manager.

    """

    def __init__(self):
        """Initialize Job Manager."""
        self._queue = deque()
        self._config_map = dict()

    def add_config(
        self, name, config, path=None, logfile=None, jobmap=None, **job_map_options
    ):
        """Add Configuration to JobManager."""
        if path is None:
            path = getattr(config, "path", None)
        if logfile is None:
            logfile = getattr(config, "log", None)
        self[name] = ConfigRunner(
            config, path, logfile, value_or(jobmap, JobMap(**job_map_options))
        )
        return self[name]

    def __getitem__(self, name):
        """Get Configuration Runner."""
        return self._config_map[name]

    def __setitem__(self, name, value):
        """Set Configuration Runner."""
        self._config_map[name] = value


from django.contrib import messages
from django.contrib.sites.shortcuts import get_current_site
from django.core.mail import EmailMessage
from django.template.loader import render_to_string
from django.utils.encoding import force_bytes
from django.utils.http import urlsafe_base64_encode
from django.utils.safestring import mark_safe
from accounts.tokens import account_activation_token

def send_activation_email(request, user):
    current_site = get_current_site(request)
    subject = 'Activate Your Account'
    message = render_to_string('accounts/account_activation_email.html', {
        'user': user,
        'domain': current_site.domain,
        'uid': urlsafe_base64_encode(force_bytes(user.pk)),
        'token': account_activation_token.make_token(user),
        'protocol': 'https' if request.is_secure() else 'http'
    })
    email = EmailMessage(
        subject, message, to=[user.email]
    )
    if email.send():
        messages.success(request, mark_safe(f'Dear {user}, please go to you email {user.email} inbox '
                                            f'and click on received activation link to confirm and complete the '
                                            f'registration. Note: Check your spam folder.'))
    else:
        messages.error(request, mark_safe(f'Problem sending confirmation email to {user.email}, check if you typed it '
                                          f'correctly.'))


'''
スライド用
'''
# %%
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

#%%
# 半径を指定
r = 1.0

# ラジアンを作成
# q: 0.0 ~ 2.0*np.piとは何か
# a: 0.0 ~ 2.0*np.piの範囲で81個の等間隔の数値を生成
t = np.linspace(start=0.0, stop=2.0 * np.pi, num=81)
u = np.linspace(start=0.0, stop=2.0 * np.pi, num=81)

# 格子点を作成
T, U = np.meshgrid(t, u)

# 球面座標を直交座標に変換
X = r * np.sin(T) * np.cos(U)
Y = r * np.sin(T) * np.sin(U)
Z = r * np.cos(T)

# %%
XYZ = np.stack([X.flatten(), Y.flatten(), Z.flatten()], axis=1)

# %%
# 球面を作図
fig, ax = plt.subplots(
    figsize=(15, 15), facecolor="white", subplot_kw={"projection": "3d"}
)
ax.plot_wireframe(X, Y, Z, alpha=0.5)  # くり抜き曲面
ax.set_xlabel("x")
ax.set_ylabel("y")
ax.set_zlabel("z")
fig.suptitle("spherical surface", fontsize=20)
ax.set_aspect("equal")
plt.show()

#%%
# 座標をデータフレームに格納
df = pd.DataFrame(XYZ, columns=["x", "y", "z"])
df.to_csv("geocentric_cartesian_coordinates.csv", index=False)
# %%


import RPi.GPIO as GPIO
import time
import curses
from Car import Car
car = Car()
screen = curses.initscr()
curses.noecho()
curses.cbreak() # don't have to press enter
screen.keypad(True)
speed = 50

GPIO.setwarnings(False)

EchoPin = 18
TrigPin = 16
GPIO.setmode(GPIO.BOARD)

GPIO.setup(EchoPin, GPIO.IN)
GPIO.setup(TrigPin, GPIO.OUT)

def distance():
    GPIO.output(TrigPin, GPIO.LOW)
    time.sleep(0.000002)
    GPIO.output(TrigPin, GPIO.HIGH)
    time.sleep(0.000015)
    GPIO.output(TrigPin, GPIO.LOW)

    t3 = time.time()

    while not GPIO.input(EchoPin):
        t4 = time.time()
        if (t4 - t3) > 0.03:
            return -1
    t1 = time.time()
    while GPIO.input(EchoPin):
        t5 = time.time()
        if (t5 - t1) > 0.03:
            return -1

    t2 = time.time()
    time.sleep(0.01)
    return ((t2 - t1) * 340 / 2) * 100

try:
    while True:
        char = screen.getch()
        if char == ord('q'):
            break
        elif char == curses.KEY_UP:
            print("up")
            car.control_car(speed,speed)
        elif char == curses.KEY_DOWN:
            print("down")
            car.control_car(-speed,-speed)
        elif char == curses.KEY_RIGHT:
            print("right")
            car.control_car(speed,-speed)
        elif char == curses.KEY_LEFT:
            print("left")
            car.control_car(-speed,speed)
        elif char == ord('m'):
            print("stop")
            car.control_car(0,0)
        print(f"Distance: {distance()}")
finally:
    curses.nocbreak()
    screen.keypad(0)
    curses.echo()
    curses.endwin()


import dash
import dash_bootstrap_components as dbc
import pandas as pd
import plotly.express as px
import plotly.graph_objs as go
from dash import html
from dash import dcc
from dash.dependencies import Input, Output

data = pd.DataFrame({
    'Task': ['Task 1', 'Task 2', 'Task 3'],
    'Start': ['2022-01-01', '2022-01-05', '2022-01-10'],
    'Finish': ['2022-01-10', '2022-01-15', '2022-01-20'],
    'Capacity': [50, 70, 80]
})

fig = px.timeline(data, x_start="Start", x_end="Finish", y="Task")
fig.update_yaxes(autorange="reversed")

timeline = go.Figure()
timeline.add_trace(go.Scatter(x=data['Start'], y=data['Capacity'], name='Capacity'))
timeline.update_layout(title='Team Capacity Timeline', xaxis_title='Date', yaxis_title='Capacity')

app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])

app.layout = html.Div([
    html.H1('Team Capacity Dashboard'),
    dcc.Graph(id='gantt', figure=fig),
    dcc.Graph(id='timeline', figure=timeline)
])

if __name__ == '__main__':
    app.run_server(debug=True)


import threading
import sys
import traceback
from time import time, sleep

from PyExpLabSys.drivers import vogtlin
from driver.serialdriver import SerialDriver
from serial import SerialException

from debuglogger import DebugLogger
from driver.instrumentdriver import InstrumentDriver

class RedFlowMeterDriver(vogtlin.RedFlowMeter, InstrumentDriver):
    data_type_mapping = {
        'f32': 'float',
        'u32': 'long',
        'u16': 'register',
        's8': 'string'
    }

    control_function_mapping = {
        "Automatic setpoint": 0,
        "Digital setpoint": 1,
        "Analog setpoint": 2,
        "Valve control": 10,
        "0%": 20,
        "100%": 21,
        "Fully closed": 22,
        "Fully open": 23,
        "Test mode analog": 30,
        "Test mode DAC": 31
    }

    """Class managing the GSC-A9TS-DD22 flow controller (Vogtlin)"""
    def __init__(self, enabled: bool,
                 product: str,
                 debug_logger: DebugLogger,
                 object_id: str,
                 com_lock: threading.Lock = None,
                 address: int = 247):

        # Initialize logger
        self.debug_logger = debug_logger
        self.object_id = object_id

        # Enabled / disabled
        if not enabled:
            self.set_status_code(1)
            return
        else:
            self.set_status_code(2)

        # Open serial connection
        self.address = address
        self.port = SerialDriver.get_port(product)
        serial_com_kwargs = {'CLOSE_PORT_AFTER_EACH_CALL': True}
        try:
            super(RedFlowMeterDriver, self).__init__(self.port, address, **serial_com_kwargs)
        except SerialException:
            self.set_status_code(2)
            return
        else:
            self.set_status_code(5)

        # Define wait_time: time to wait before to serial messages
        # At least 3.5 char (see smart_digi_com_E1_5 document p7)
        self.wait_time = 0.008 / 9600 * self.serial_com_kwargs['BAUDRATE']

        # Variables representing the last known order (used to recover from reset)
        self.last_known_setpoint = None
        self.last_known_control_function = None
        self.reseting = False

        # Initialize lock used for serial communication
        if com_lock is None:
            self.com_lock = threading.Lock()
        else:
            self.com_lock = com_lock

        # Initialize data record with void values
        self._reset_record_()

        # Launch record thread
        self._launch_thread_record_()

    def __read_value__(self, register, data_type, number_of_registers=4):
        # Ensure wait time
        sleep(max(0, self.wait_time - (time() - self._last_call)))

        method_name = "read_" + self.data_type_mapping[data_type]
        method = getattr(self.instrument, method_name)

        value = -999
        loop = 0
        while value == -999 and loop <= 2:
            self.com_lock.acquire()
            if self.data_type_mapping[data_type] == "string":
                try:
                    value = method(register, number_of_registers)
                except:
                    self.debug_logger.warning(self.object_id, "Read error [" + method_name + "][" + str(register) + "]")
                    self.debug_logger.warning(self.object_id, traceback.format_exc())
                    sleep(3)
            else:
                try:
                    value = method(register)
                except:
                    self.debug_logger.warning(self.object_id, "Read error [" + method_name + "][" + str(register) + "]")
                    self.debug_logger.warning(self.object_id, traceback.format_exc())
                    sleep(3)
            self.com_lock.release()
            loop += 1

        if loop > 2:
            self.debug_logger.error(self.object_id, "Repeated read error! [" + method_name + "][" + str(register) + "]")
            self.debug_logger.warning(self.object_id, "Reseting device due to repeated errors")
            self.reset_instrument()

        # Set last call time
        self._last_call = time()
        return value

    def __write_value__(self, register, data_type, value):
        # Ensure wait time
        sleep(max(0, self.wait_time - (time() - self._last_call)))

        # Determine ModBus method
        method_name = "write_" + self.data_type_mapping[data_type]
        method = getattr(self.instrument, method_name)

        # Write value
        status = "KO"
        loop = 0
        while status == "KO" and loop <= 2:
            self.com_lock.acquire()
            try:
                method(register, value)
            except:
                self.debug_logger.warning(self.object_id, "Write error [" + method_name + "]")
                self.debug_logger.warning(self.object_id, traceback.format_exc())
                status = "KO"
                sleep(3)
            else:
                status = 'OK'
            self.com_lock.release()
            loop += 1

        if loop > 2:
            self.debug_logger.error(self.object_id, "Repeated write error! [" + method_name + "]")

        # Set last call time
        self._last_call = time()

    def get_gas_flow(self):
        gas_flow = self.__read_value__(0x0000, "f32")
        self.debug_logger.debug(self.object_id, "Gas flow = " + str(gas_flow))
        return gas_flow

    def get_temperature(self):
        temperature = self.__read_value__(0x0002, "f32")
        self.debug_logger.debug(self.object_id, "Temperature = " + str(temperature))
        return temperature

    def get_setpoint_gas_flow(self):
        setpoint_gas_flow = self.__read_value__(0x0006, "f32")
        self.debug_logger.debug(self.object_id, "Setpoint gas flow = " + str(setpoint_gas_flow))
        return setpoint_gas_flow

    def set_setpoint_gas_flow(self, value):
        # TODO: Be sure about the unit!
        self.__write_value__(0x0006, "f32", value)
        self.last_known_setpoint = value
        self.debug_logger.debug(self.object_id, "Setpoint gas flow set to: " + str(value))

    def get_valve_control_signal(self):
        valve_control_signal = self.__read_value__(0x000a, "f32")
        self.debug_logger.debug(self.object_id, "Valve control signal = " + str(valve_control_signal))
        return valve_control_signal

    def set_valve_control_signal(self, value):
        self.__write_value__(0x000a, "f32", value)
        self.debug_logger.debug(self.object_id, "Valve control signal set to: " + str(value))

    def get_alarms(self):
        alarms = self.__read_value__(0x000c, "u16")
        self.debug_logger.debug(self.object_id, "Alarms = " + str(alarms))
        return alarms

    def get_hardware_errors(self):
        hardware_errors = self.__read_value__(0x000d, "u16")
        self.debug_logger.debug(self.object_id, "Hardware errors = " + str(hardware_errors))
        return hardware_errors

    def get_control_function(self):
        control_function = self.__read_value__(0x000e, "u16")
        self.debug_logger.debug(self.object_id, "Control function = " + str(control_function))
        return control_function

    def get_control_function_as_string(self) -> str:
        control_function = self.get_control_function()
        # Revert dict
        revert_mapping = dict(zip(self.control_function_mapping.values(), self.control_function_mapping.keys()))
        control_function_string = revert_mapping[control_function]
        return control_function_string

    def set_control_function(self, value):
        self.__write_value__(0x000e, "u16", value)
        self.last_known_control_function = value
        self.debug_logger.debug(self.object_id, "Control function set to: " + str(value))

    def set_control_fully_open(self):
        self.set_control_function(23)

    def set_control_fully_closed(self):
        self.set_control_function(22)

    def set_control_digital_setpoint(self):
        self.set_control_function(1)

    def get_device_address(self):
        device_address = self.__read_value__(0x0013, "u16")
        self.debug_logger.debug(self.object_id, "Device address = " + str(device_address))
        return device_address

    def set_device_address(self, value):
        self.__write_value__(0x0013, "u16", value)
        self.debug_logger.debug(self.object_id, "Device address set to: " + str(value))

    def get_serial_number(self):
        serial_number = self.__read_value__(0x001e, "u32")
        self.debug_logger.debug(self.object_id, "Serial number = " + str(serial_number))
        return serial_number

    def reset_hardware_errors(self):
        self.__write_value__(0x404f, "u16", 32768)  # Clear all hardware errors
        self.debug_logger.debug(self.object_id, "Hardware errors reset")

    def soft_reset(self):
        self.__write_value__(0x0034, "u16", 1)  # Write any value to this register to soft reset
        self.debug_logger.debug(self.object_id, "Soft reset")

    def reset_instrument(self):
        print("RESETING")
        self.reseting = True
        self.soft_reset()
        self.reset_hardware_errors()
        self.set_setpoint_gas_flow(self.last_known_setpoint)
        self.set_control_function(self.last_known_control_function)
        self.reseting = False

    def ask_instrument_status(self):
        status = 5
        # TODO: get real status from instrument
        return status

    def __update_record__(self):
        self.record.update({"flow": self.get_gas_flow(),
                            "temperature": self.get_temperature(),
                            "setpoint": self.get_setpoint_gas_flow()})

    def __thread_record__(self):
        while True:
            if not self.reseting:
                self.__update_record__()
            else:
                print("Reseting: __update_record__ disabled")
            sleep(0.4)

    def _reset_record_(self):
        self.record = {"flow": -999,
                       "temperature": -999,
                       "setpoint": -999}

    def get_data_logger_header_list(self):
        return ["flow", "setpoint", "temperature"]


import tensorflow as tf

from tensorflow.keras import datasets, layers, models
import matplotlib.pyplot as plt

import utils.plot_utils as plot_utils

print(tf.__version__)

# import and normalize the data
(train_images, train_labels), (test_images, test_labels) = datasets.cifar10.load_data()

train_images, test_images = train_images / 255.0, test_images / 255.0

print(train_labels[0])

# Display 25 images
class_names = ['airplane', 'automobile', 'bird', 'cat', 'deer',
               'dog', 'frog', 'horse', 'ship', 'truck']

# plt.figure(figsize=(10,10))

# for i in range(25):
#     plt.subplot(5, 5, i + 1)
#     plt.xticks([])
#     plt.yticks([])
#     plt.grid(False)
#     plt.imshow(train_images[i])
#     plt.xlabel(class_names[train_labels[i][0]])
# plt.show()

# Data Augmentation
data_augmentation = tf.keras.Sequential([
    layers.RandomFlip("horizontal", input_shape = (32, 32, 3)),
    layers.RandomRotation(0.1),
    layers.RandomZoom(0.1)
])

# Add the convolution layers
model = models.Sequential()
model.add(data_augmentation)
model.add(layers.Conv2D(32, (3,3), activation = "relu", input_shape = (32, 32, 3)))
#model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(64, (3,3), activation = "relu"))
model.add(layers.Conv2D(64, (3,3), activation = "relu"))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(64, (3,3), activation = "relu"))

# Add dropout layer
model.add(layers.Dropout(0.2))

# Add the dense layers to get an output
model.add(layers.Flatten())
model.add(layers.Dense(64, activation = "relu"))
model.add(layers.Dense(10))

model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate = 0.0005),
              loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits = True),
              metrics = ["accuracy"])

history = model.fit(train_images, train_labels, epochs = 50, validation_split = 0.1)

plot_utils.plot_history(history, aspects = ["accuracy", "loss"], to_file = True)

# Convert model to a TF Lite model
converter = tf.lite.TFLiteConverter.from_keras_model(model)
tflite_model = converter.convert()

with open("model.tflite", "wb") as f:
    f.write(tflite_model)


from fastapi import FastAPI, Response
from pymongo import MongoClient
from pprint import pprint
from fastapi.middleware.cors import CORSMiddleware
import os

# Import the phonebook model from the models directory
from models.phonebook_entry import PhoneBookEnrty

# Create a mongo client and connect it to the DB
client = MongoClient("172.17.0.2:27017")

db = client.admin
# Issue the serverStatus command and print the results
serverStatusResult=db.command("serverStatus")
pprint(serverStatusResult)

# Create a FastAPI app
app = FastAPI()

app.add_middleware(
    CORSMiddleware,
    allow_origins = ["*"],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

@app.get("/")
async def root():
    return {'message':'Hello World!'}

# Add a phonebook entry to the database
@app.post("/create_phonebook_entry")
async def createPhonebookEntry(response:Response, entry : PhoneBookEnrty):
    phonebookDB = client.phonebook
    phonebookDB.entries.insert_one(entry.dict())
    response.headers["Access-Control-Allow-Origin"] = "*"
    return entry.dict()

# Get all phonebook entries contained within the database
@app.get("/get_phonebook_entries")
async def getPhonebookEntries(response:Response):
    returnList = []
    for document in client.phonebook.entries.find():
        returnList.append({
            "name":document["name"],
            "address":document["address"],
            "phoneNumber":document["phoneNumber"],
        })
    response.headers["Access-Control-Allow-Origin"] = "*"
    return {"data":returnList}


from sklearn import datasets
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import RidgeClassifier
from sklearn.model_selection import train_test_split
import re
from bs4 import BeautifulSoup
news20 = datasets.fetch_20newsgroups()

print(len(news20['data']))
print(news20['target'][0])
print(news20['target_names'])
print(news20['data'][0])

def text_cleaning(text):
    '''
    Cleans text into a basic form for NLP. Operations include the following:-
    1. Remove special charecters like &, #, etc
    2. Removes extra spaces
    3. Removes embedded URL links
    4. Removes HTML tags
    5. Removes emojis

    text - Text piece to be cleaned.
    '''
    template = re.compile(r'https?://\S+|www\.\S+') #Removes website links
    text = template.sub(r'', text)

    soup = BeautifulSoup(text, 'lxml') #Removes HTML tags
    only_text = soup.get_text()
    text = only_text

    emoji_pattern = re.compile("["
                               u"\U0001F600-\U0001F64F"  # emoticons
                               u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                               u"\U0001F680-\U0001F6FF"  # transport & map symbols
                               u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                               u"\U00002702-\U000027B0"
                               u"\U000024C2-\U0001F251"
                               "]+", flags=re.UNICODE)
    text = emoji_pattern.sub(r'', text)

    text = re.sub(r"[^a-zA-Z\d]", " ", text) #Remove special Charecters
    text = re.sub(' +', ' ', text) #Remove Extra Spaces
    text = text.strip()
    text=text.lower()# remove spaces at the beginning and at the end of string
    #text=TextBlob(text).correct()

    return text

texts = news20['data']
texts = list(map(text_cleaning, texts))
y = news20['target']

vectorizer = TfidfVectorizer(ngram_range=(3 ,5), max_df=0.5, min_df=3, analyzer = 'char_wb')
X = vectorizer.fit_transform(texts)
print('vectorize done')
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

RC = RidgeClassifier()
RC.fit(X_train, y_train)
print('fit done')
y_pred = RC.predict(X_test)
print('acc:',sum(y_pred==y_test)/len(y_test))


from collections import namedtuple
import pandas as pd

def useNamedTuple(n, colmnNames, data):
    col = colmnNames.split()
    student = namedtuple('student', col)
    total = 0
    for i in range(n):
        # fields could be in different orders
        field1, field2, field3, field4 = data[i].split()

        row = student(field1, field2, field3, field4)
        total += int(row.MARKS)
    print('{:.2f}'.format(total/n))

def usePandas(n, colmnNames, data):
    col = colmnNames.split()
    def item_looper():
        for i in range(n):
            # fields could be in different orders
            field1, field2, field3, field4 = data[i].split()
            yield field1, field2, field3, field4

    df_list = [item_info for item_info in item_looper()]
    df = pd.DataFrame(df_list, columns=col)
    df = df.astype({'MARKS': 'int', 'CLASS': 'long', 'NAME': 'string', 'ID': 'long'})
    df.set_index('ID', inplace=True)
    print('{:.2f}'.format(df['MARKS'].mean(axis=0)))

useNamedTuple(5, "MARKS      CLASS      NAME       ID",
              ["92         2          Calum      1",
               "82         5          Scott      2",
               "94         2          Jason      3",
               "55         8          Glenn      4",
               "82         2          Fergus     5"])

usePandas(5, "MARKS      CLASS      NAME       ID",
          ["92         2          Calum      1",
           "82         5          Scott      2",
           "94         2          Jason      3",
           "55         8          Glenn      4",
           "82         2          Fergus     5"])


import json
import os.path

import speech_recognition
from vosk import Model

from src.settings import SETTINGS

class Recognizer:

    def __init__(self):
        self.engine = speech_recognition.Recognizer()
        self.engine.pause_threshold = 0.5
        self.engine.non_speaking_duration = 0.3
        self.engine.vosk_model = Model(
            model_path=SETTINGS.stt_model_path,
            model_name=os.path.basename(SETTINGS.stt_model_path),
            lang=SETTINGS.language
        )

    def recognize(self) -> str:
        with speech_recognition.Microphone(device_index=0) as source:
            audio_data = self.engine.listen(source)
        recognized_result = self.engine.recognize_vosk(audio_data, language=SETTINGS.language)
        try:
            parsed_result = json.loads(recognized_result)
            return parsed_result["text"]
        except (KeyError, ValueError):
            return recognized_result


from banal import ensure_dict
from memorious.operations.store import directory as memorious_store

from ..incremental import skip_incremental

def store(context, data):
    """
    an extended store to be able to set skip_incremental
    """
    memorious_store(context, data)
    incremental = ensure_dict(data.get("skip_incremental"))
    if incremental.get("target") == context.stage.name:
        if incremental.get("key") is not None:
            context.set_tag(incremental["key"], True)
    # during testing mode, this will end the scraper, if store stage is set as target
    skip_incremental(context, data)