import inspect
import os
import numpy as np
import pickle
import regex

import tensorflow as tf
from tensorflow import keras
from transformers import BertConfig, TFBertMainLayer

from .utils import found_package
import transformers
import tensorflow_addons as tfa

from .utils import iobes_iob, parse_lr_method
from .conlleval_perl import evaluate_lines
from .layers import SimpleSequenceScores, RNNSequenceScores, CharacterFeatures
from .losses import PlainLoss
from .loader import get_input_format, dict_to_tf_dataset
from . import bert_optimization

__author__ = 'Tanjin He'
__maintainer__ = 'Tanjin He, Ziqin (Shaun) Rong'
__email__ = 'tanjin_he@berkeley.edu, rongzq08@gmail.com'

# Modified based on the NER Tagger code from arXiv:1603.01360 [cs.CL]

class NERModel(keras.Model):
    """
    Network architecture.
    """

    def __init__(self,
                 model_path=None,
                 model_name=None,
                 to_reload_model=False,
                 tag_scheme='iobes',
                 clean_tag=True,
                 id_to_word=None,
                 id_to_char=None,
                 id_to_tag=None,
                 classifier_type='lstm',
                 bert_path=None,
                 bert_first_trainable_layer=0,
                 word_dim=100,
                 word_lstm_dim=100,
                 word_bidirect=True,
                 word_unroll=False,
                 word_rnn_wrapper=False,
                 char_dim=25,
                 char_lstm_dim=25,
                 char_bidirect=True,
                 char_combine_method='concat',
                 char_unroll=False,
                 char_rnn_wrapper=False,
                 # cap_dim,
                 ele_num=True,
                 only_CHO=True,
                 tar_tag=False,
                 pre_tag=False,
                 # topic_dim,
                 # keyword_dim,
                 rnn_type='gru',
                 lower=False,
                 zeros=False,
                 use_ori_text_char=False,
                 crf=True,
                 crf_begin_end=True,
                 dropout=0.5,
                 pre_embedding=None,
                 lr_method='sgd-lr_.005',
                 loss_per_token=False,
                 batch_size=1,
                 num_epochs=100,
                 steps_per_epoch=3500,
                 **kwargs):
        """
        Initialize the model. We can init a empty model with model_name, or reload
        a pre-trained model from model_path

        :param model_path: File path to reload the model. If specified, model will be
        reloaded from model_path, and model_name will be discarded.
        :param model_name: Name of the model. If specified, the model will save in a
        folder called model_name.
        """
        super().__init__(**kwargs)
        # Model location
        parent_folder = os.path.abspath(
            os.path.join(
                os.path.dirname(__file__),
                '..'
            )
        )
        if not model_path:
            if model_name:
                self.model_path = os.path.join(parent_folder, 'generated',  model_name)
            else:
                self.model_path = os.path.join(parent_folder, 'generated', 'model_1')
            if not os.path.exists(self.model_path):
                os.makedirs(self.model_path)
            self.parameters_path = os.path.join(self.model_path, 'parameters.pkl')
        else:
            # reload model parameters
            self.model_path = model_path
            self.parameters_path = os.path.join(self.model_path, 'parameters.pkl')

        self.last_cp_dir = os.path.join(self.model_path, 'last_cp')
        if not (to_reload_model or os.path.exists(self.last_cp_dir)):
            os.makedirs(self.last_cp_dir)
        self.last_cp_path = os.path.join(self.last_cp_dir, 'cp.ckpt')
        self.opt_cp_dir = os.path.join(self.model_path, 'opt_cp')
        if not (to_reload_model or os.path.exists(self.opt_cp_dir)):
            os.makedirs(self.opt_cp_dir)
        self.opt_cp_path = os.path.join(self.opt_cp_dir, 'cp.ckpt')

        self.tag_scheme = tag_scheme
        self.clean_tag = clean_tag
        self.classifier_type = classifier_type
        self.bert_path = bert_path
        self.bert_first_trainable_layer = bert_first_trainable_layer
        self.word_dim = word_dim
        self.word_lstm_dim = word_lstm_dim
        self.word_bidirect = word_bidirect
        self.word_unroll = word_unroll
        self.word_rnn_wrapper = word_rnn_wrapper
        self.char_dim = char_dim
        self.char_lstm_dim = char_lstm_dim
        self.char_bidirect = char_bidirect
        self.char_combine_method = char_combine_method
        self.char_unroll = char_unroll
        self.char_rnn_wrapper = char_rnn_wrapper
        self.ele_num = ele_num
        self.only_CHO = only_CHO
        self.tar_tag = tar_tag
        self.pre_tag = pre_tag
        self.rnn_type = rnn_type
        self.lower = lower
        self.zeros = zeros
        self.use_ori_text_char = use_ori_text_char
        self.crf = crf
        self.crf_begin_end = crf_begin_end
        self.dropout = dropout
        self.pre_embedding = pre_embedding
        self.lr_method = lr_method
        self.loss_per_token = loss_per_token
        self.batch_size = batch_size
        self.num_epochs = num_epochs
        self.steps_per_epoch = steps_per_epoch
        assert id_to_tag
        if not self.bert_path:
            assert id_to_word and id_to_char and id_to_tag
        self.create_mappings(
            id_to_word=id_to_word,
            id_to_char=id_to_char,
            id_to_tag=id_to_tag,
        )
        if not to_reload_model:
            # TODO: need to think about how to save bert model
            self.save_model()

        # we will use mask defaultly
        # therefore, id of words and tags should starts from 1
        # add embedding for mask
        if self.pre_embedding is not None:
            self.word_dim = np.shape(self.pre_embedding)[1]
            self.pre_embedding_matrix = np.concatenate(
                (
                    np.zeros((1, self.word_dim)),
                    self.pre_embedding
                )
            )

        lr_method_name, lr_method_parameters = parse_lr_method(self.lr_method)
        print('lr_method_name, lr_method_parameters',
              lr_method_name, lr_method_parameters)
        if lr_method_name == 'sgd':
            self.optimizer = keras.optimizers.SGD(
                learning_rate=lr_method_parameters.get('lr', 0.005),
            )
        elif lr_method_name == 'adam':
            self.optimizer = keras.optimizers.Adam(
                learning_rate=lr_method_parameters.get('lr', 5e-5),
            )
        elif lr_method_name == 'adamdecay':
            num_train_steps = self.steps_per_epoch * self.num_epochs
            num_warmup_steps = int(
                num_train_steps * lr_method_parameters.get('warmup', 0.05)
            )
            print('num_train_steps, num_warmup_steps',
                  num_train_steps, num_warmup_steps)
            self.optimizer = bert_optimization.create_optimizer(
                init_lr=lr_method_parameters.get('lr', 5e-5),
                num_train_steps=num_train_steps,
                num_warmup_steps=num_warmup_steps,
                epsilon=lr_method_parameters.get('epsilon', 1e-6),
                weight_decay_rate=lr_method_parameters.get('weight_decay_rate', 0.01),
            )
        else:
            raise ValueError(
                "Not implemented learning method: {}".format(lr_method_name)
            )

        self.loss = PlainLoss(name='loss_layer')

        self.create_constants()
        # add layers
        if self.bert_path:
            if to_reload_model:
                bert_config = BertConfig.from_pretrained(self.bert_path)
                self.bert_layer = TFBertMainLayer(bert_config, name="bert")
            else:
                self.bert_layer = transformers.TFBertModel.from_pretrained(
                    self.bert_path,
                    from_pt=True,
                ).bert
            self.set_bert_trainable_layers(self.bert_layer, self.bert_first_trainable_layer)

            assert self.word_dim == 0
            assert self.char_dim == 0
            assert pre_embedding == None

        if self.pre_embedding is not None:
            # init embedding layer
            # https://stackoverflow.com/questions/47057361/keras-using-tensorflow-backend-masking-on-loss-function
            # mapping ids always start from 1
            # To use mapping directly, mask_zero should always be ture
            self.word_layer = keras.layers.Embedding(
                input_dim=self.n_words+1,
                output_dim=self.word_dim,
                embeddings_initializer=keras.initializers.Constant(
                    self.pre_embedding_matrix
                ),
                mask_zero=True,
                trainable=True,
                name='word_layer',
            )
        elif self.word_dim:
            self.word_layer = keras.layers.Embedding(
                input_dim=self.n_words+1,
                output_dim=self.word_dim,
                embeddings_initializer='glorot_uniform',
                mask_zero=True,
                trainable=True,
                name='word_layer',
            )

        if self.dropout:
            self.dropout_layer = keras.layers.Dropout(self.dropout)

        # TODO: it is better to assign a name to each layer in the future
        if self.char_dim:
            self.char_layer = CharacterFeatures(
                n_chars=self.n_chars,
                char_dim=self.char_dim,
                char_lstm_dim=self.char_lstm_dim,
                char_bidirect=self.char_bidirect,
                char_combine_method=self.char_combine_method,
                batch_size=self.batch_size,
                rnn_type=self.rnn_type,
                unroll=self.char_unroll,
                rnn_wrapper=self.char_rnn_wrapper,
            )

        if self.classifier_type == 'simple':
            print('classifier_type', classifier_type)
            # activation function here is always None
            self.unary_score_layer = SimpleSequenceScores(
                n_tags=self.n_tags,
            )
        elif self.classifier_type == 'lstm':
            print('classifier_type', classifier_type)
            self.unary_score_layer = RNNSequenceScores(
                word_bidirect=self.word_bidirect,
                word_lstm_dim=self.word_lstm_dim,
                n_tags=self.n_tags,
                rnn_type=self.rnn_type,
                unroll=self.word_unroll,
                rnn_wrapper=self.word_rnn_wrapper,
            )
        else:
            raise NotImplementedError('Classifier {} not recognized!'.format(self.classifier_type))

        self.compile(
            optimizer=self.optimizer,
            loss=self.loss,
        )

    def create_mappings(self, id_to_word, id_to_char, id_to_tag):
        """
        We need to save the mappings if we want to use the model later.

        :param id_to_word: mapping from a number (id) to a word in text
        :param id_to_tag: mapping from a number (id) to a tag of word
        """
        self.id_to_word = id_to_word
        self.id_to_char = id_to_char
        self.id_to_tag = id_to_tag

        if self.id_to_word:
            self.n_words = len(self.id_to_word)
        if self.id_to_char:
            self.n_chars = len(self.id_to_char)

        self.tag_to_id = {v: k for (k, v) in self.id_to_tag.items()}
        self.all_tags = np.array(sorted(
            self.tag_to_id.keys(),
            key=lambda x: self.tag_to_id[x]
        ))

        self.all_tag_ids = np.array(
            [self.tag_to_id[t] for t in self.all_tags],
            dtype=np.int32
        )
        self.n_tags = len(self.all_tags)
        self.tag_tag_to_idx = {
            t: i for (i, t) in enumerate(self.all_tags)
        }
        self.tag_id_to_idx = {
            id: i for (i, id) in enumerate(self.all_tag_ids)
        }
        self.tag_id_to_idx_lookup = tf.lookup.StaticHashTable(
            initializer=tf.lookup.KeyValueTensorInitializer(
                keys=self.all_tag_ids,
                values=tf.range(self.n_tags, dtype=tf.int32),
            ),
            default_value=tf.constant(0, dtype=tf.int32),
            name="tag_id_to_idx_lookup",
        )
        # attention: string in tensor is bytes
        # need to use x.decode('utf-8') after lookup
        self.tag_id_to_tag_lookup = tf.lookup.StaticHashTable(
            initializer=tf.lookup.KeyValueTensorInitializer(
                keys=self.all_tag_ids,
                values=self.all_tags,
            ),
            default_value=tf.constant('<MASK>'),
            name="tag_id_to_tag_lookup",
        )

    def set_bert_trainable_layers(self, bert_main_layer, bert_first_trainable_layer=0):
        # get name of encoder layers
        pattern_bert_layer = regex.compile('.*(layer_._[0-9]+).*')
        bert_encoder_layer_ids = []
        for w in bert_main_layer.encoder.weights:
            tmp_m = pattern_bert_layer.match(w.name)
            if tmp_m and tmp_m.group(1) not in bert_encoder_layer_ids:
                bert_encoder_layer_ids.append(tmp_m.group(1))

       # set bert layers trainable and not trainable
        bert_trainable_layers = bert_encoder_layer_ids[bert_first_trainable_layer:]
        pattern_bert_trainable_layer = regex.compile(
            '.*encoder.+({}).*'.format('|'.join(bert_trainable_layers))
        )
        for w in bert_main_layer.weights:
            if (
                len(bert_trainable_layers) > 0
                and pattern_bert_trainable_layer.match(w.name)
            ):
                w._trainable = True
            else:
                w._trainable = False
            # only for test
            if bert_first_trainable_layer == 100:
                w._trainable = True

    def create_constants(self):
        self.small = -1000.0

        if self.crf and self.crf_begin_end:
            self.b_s = np.ones((1, self.n_tags + 2,), dtype=np.float32) * self.small
            self.b_s[:, -2] = 0.0
            self.e_s = np.ones((1, self.n_tags + 2,), dtype=np.float32) * self.small
            self.e_s[:, -1] = 0.0

            self.b_idx = np.array([self.n_tags], dtype=np.int32)
            self.e_idx = np.array([self.n_tags+1], dtype=np.int32)

    def build(self, input_shape):
        """

        :param input_shape: (None, num_eles)
        :return:
        """
        if self.crf_begin_end:
            self.transitions = self.add_weight(
                shape=(self.n_tags+2, self.n_tags+2),
                initializer='glorot_uniform',
                trainable=True,
                name='transitions'
            )
        else:
            self.transitions = self.add_weight(
                shape=(self.n_tags, self.n_tags),
                initializer='glorot_uniform',
                trainable=True,
                name='transitions'
            )

    def call(self, inputs, training=None):
        # preprocess inputs
        processed_inputs = self.prepare_final_inputs(
            inputs, training=training
        )
        _x = processed_inputs['final_inputs']
        word_mask_bool = processed_inputs['word_mask_bool']
        tags_idxes = processed_inputs['tags_idxes']
        sents_len = processed_inputs['sents_len']
        sents_num_tokens = processed_inputs['sents_num_tokens']
        word_mask_float = tf.cast(word_mask_bool, tf.float32)
        # layers
        # tags_scores: (None, seq_len, n_tags)
        tags_scores = self.get_tags_scores(
            final_inputs=_x,
            word_mask_bool=word_mask_bool,
            word_mask_float=word_mask_float,
        )

        # calculate loss
        if self.crf:
            # TODO: it is better to use scatter_nd to rearrange the scores
            #  to drop out masked elements for bert, which makes crf easier
            if self.crf_begin_end:
                observations = self.add_begin_end_score(
                    tags_scores,
                    sents_len,
                    word_mask_float,
                )
                padded_tags_idxes = self.add_begin_end_idx(
                    tags_idxes,
                    sents_len
                )
                padded_sents_len = sents_len + 2
            else:
                observations = tags_scores
                padded_tags_idxes = tags_idxes
                padded_sents_len = sents_len

            # crf loss
            log_likelihood, transition_params = tfa.text.crf_log_likelihood(
                inputs=observations,
                tag_indices=padded_tags_idxes,
                sequence_lengths=padded_sents_len,
                transition_params=self.transitions,
            )
            loss = -log_likelihood
        else:
            # there is already "-" in keras.losses classes
            loss = keras.losses.sparse_categorical_crossentropy(
                y_true=tags_idxes,
                y_pred=tags_scores,
                from_logits=True,
            )
            loss = loss * word_mask_float
            loss = tf.reduce_sum(loss, axis=-1)

        # # TODO: if mean is better, do we need to weight by sent len?
        if self.loss_per_token:
            loss = loss/tf.cast(sents_num_tokens, tf.float32)

        return loss

    def prepare_final_inputs(self, inputs, training=None):
        # # save parameters
        # if training:
        #     saved_locals = locals()
        #     self.parameters = saved_locals
        #     self.parameters.update(self.parameters['kwargs'])
        #     del self.parameters['kwargs']
        #     del self.parameters['self']
        #     # not save pre_emb because it might be very large
        #     # and duplicates the original embedding file
        #     del self.parameters['pre_emb']
        #     del self.parameters['training']
        #     with open(self.parameters_path, 'wb') as f:
        #         pickle.dump(self.parameters, f)

        # word_ids: (None, max_seq_len, )
        word_ids = inputs['words']
        # word_mask_bool: (None, max_seq_len, )
        word_mask_bool = tf.cast(inputs['score_mask'], tf.bool)

        # Final input (all word features)
        _x = []

        # bert_path
        if self.bert_path:
            # note: return_dict=True will be applied by default
            # it is commeted out because TF says "The parameter `return_dict`
            # cannot be set in graph mode and will always be set to `True`."
            bert_output = self.bert_layer(
                word_ids,
                attention_mask=inputs['bert_attention_mask'],
                token_type_ids=inputs['bert_token_type_ids'],
                # return_dict=True
            )
            _x.append(bert_output.last_hidden_state)

        # word dim
        if self.word_dim:
            # attention! word_ids start from 1, and mask_zero==True
            # thus, they are consistent
            # otherwise, a lookup table is needed to convert id to idx in embeddings
            word_input = self.word_layer(word_ids)
            _x.append(word_input)

        # char dim
        if self.char_dim:
            char_ids = inputs['chars']
            char_input = self.char_layer(char_ids)
            _x.append(char_input)

        if self.ele_num:
            ele_num = tf.expand_dims(inputs['ele_num'], axis=2)
            _x.append(ele_num)

        if self.only_CHO:
            only_CHO = tf.expand_dims(
                tf.cast(inputs['only_CHO'], tf.float32),
                axis=2
            )
            _x.append(only_CHO)

        if self.tar_tag:
            # only for coding test
            tar_tag = tf.expand_dims(
                tf.cast(inputs['tar_tag'], tf.float32),
                axis=2
            )
            _x.append(tar_tag)

        if self.pre_tag:
            # only for coding test
            pre_tag = tf.expand_dims(
                tf.cast(inputs['pre_tag'], tf.float32),
                axis=2
            )
            _x.append(pre_tag)

        # Prepare final input
        if len(_x) == 1:
            final_inputs = _x[0]
        else:
            final_inputs = tf.concat(_x, axis=-1)

        if self.dropout:
            final_inputs = self.dropout_layer(final_inputs, training=training)

        # get final_y, which is tags_idxes used to cal cost
        tag_ids = inputs['tags']
        tags_idxes = self.tag_id_to_idx_lookup.lookup(tag_ids)

        # get sentence length
        sents_len = tf.reshape(inputs['length'], (-1,))

        # get number of tokens in batched sentences
        sents_num_tokens = tf.reshape(inputs['num_tokens'], (-1,))

        processed_inputs = {
            'final_inputs': final_inputs,
            'word_mask_bool': word_mask_bool,
            'tags_idxes': tags_idxes,
            'sents_len': sents_len,
            'sents_num_tokens': sents_num_tokens,
        }
        return processed_inputs

    def get_tags_scores(self, final_inputs, word_mask_bool, word_mask_float):
        """
        return tags_scores
        if crf, scores are logits
        if not crf, scores are normalized with softmax

        :param final_inputs:
        :param word_mask_bool:
        :param word_mask_float:
        :return: (None, max_seq_len, n_tags)
        """
        # layers to get scores before crf
        # the score here is not softmax normalized
        tags_scores = self.unary_score_layer(final_inputs, mask=word_mask_bool)
        tags_scores = tags_scores * tf.expand_dims(word_mask_float, axis=2)
        # If not use crf, normalized tags_scores with softmax to get probability
        # at this time, we should set logits=False in sparse_categorical_crossentropy
        # This is same as not use softmax and set logits=True in
        # sparse_categorical_crossentropy. The loss is not changed
        # ref: https://github.com/tensorflow/tensorflow/blob/1cf0898dd4331ba
        # f93fe77205550f2c2e6c90ee5/tensorflow/python/keras/backend.py#L4508
        return tags_scores

    def pad_with_begin_end(self, seq, sentence_len, begin, end):
        """

        :param seq: (max_seq_len, ) or (max_seq_len, dim)
        :param sentence_len: int, real seq len
        :return:
        """
        padded_seq = tf.concat(
            [
                begin,
                seq[: sentence_len],
                end,
                seq[sentence_len:]
            ],
            axis=0
        )
        return padded_seq

    def add_begin_end_score(self,
                            tags_scores,
                            sents_len,
                            word_mask_float):
        """
        add begin/end tag as the last 2 dimensions in tags_scores

        :param tags_scores: (batch_size, max_seq_len, n_tags)
                    or (batch_size=1, real_seq_len, n_tags)
        :param sents_len: (batch_size, )
                    or (batch_size=1, )
                    consistent with tags_score
        :param word_mask_float: (batch_size, max_seq_len)
                    or (batch_size=1, real_seq_len)
                    consistent with tags_score
        :return: (batch_size, max_seq_len+2, n_tags+2)
        """

        observations = tf.concat(
            [
                tags_scores,
                tf.tile(
                    tf.expand_dims(word_mask_float, 2),
                    multiples=[1, 1, 2]
                )*self.small,
            ],
            axis=-1,
        )

        observations, _ = tf.map_fn(
            lambda x: (
                self.pad_with_begin_end(x[0], x[1], self.b_s, self.e_s),
                x[1]
            ),
            (observations, sents_len)
        )

        return observations

    def add_begin_end_idx(self, tags_idxes, sents_len):
        padded_tags_idxes, _ = tf.map_fn(
            lambda x: (
                self.pad_with_begin_end(x[0], x[1], self.b_idx, self.e_idx),
                x[1]
            ),
            (tags_idxes, sents_len)
        )
        return padded_tags_idxes

    def predict(self, x_batches, return_tensor=False, **kwargs):
        """
        return tags_scores, each sentence would be in real length
        rather than the padded ones
        Scores are logits rather than probabilities

        :param x_batches: all data in multiple batches
        :param kwargs:
        :return:
        """
        # here only scores without crf considered
        # use  predict_idx to get the ids
        processed_inputs = x_batches.map(self.prepare_final_inputs)
        scores = processed_inputs.map(
            lambda x: self.get_tags_scores(
                final_inputs=x['final_inputs'],
                word_mask_bool=x['word_mask_bool'],
                word_mask_float=tf.cast(x['word_mask_bool'], tf.float32),
            )
        ).unbatch()
        word_mask = processed_inputs.map(
            lambda x: x['word_mask_bool']
        ).unbatch()
        scores_mask = tf.data.Dataset.zip((scores, word_mask))
        # restore the length from max_seq_len to real_seq_len
        scores = scores_mask.map(lambda s, mask: s[mask])
        # return tensor for further calculation with tf ops
        # if return tensor and crf==True, we return observations here
        # rather than scores for convenience in furthere calculation
        if not return_tensor:
            scores = [s.numpy() for s in scores]
        return scores

    def predict_idx(self, x_batches, **kwargs):
        # here scores_sents is unbatched data
        scores_sents = self.predict(x_batches, return_tensor=True, **kwargs)

        if self.crf:
            if self.crf_begin_end:
                observations = scores_sents.batch(1)
                observations = observations.map(
                    lambda x: self.add_begin_end_score(
                        tags_scores=x,
                        sents_len=tf.shape(x)[1:2],
                        word_mask_float=tf.ones((1, tf.shape(x)[1])),
                    )
                ).unbatch()
            else:
                observations = scores_sents
            observations = [s.numpy() for s in observations]
            tag_idx_sents = list(map(
                lambda x: (tfa.text.crf.viterbi_decode(
                    x,
                    self.transitions
                ))[0],
                observations
            ))
            if self.crf_begin_end:
                tag_idx_sents = list(map(
                    lambda x: x[1:-1],
                    tag_idx_sents
                ))
        else:
            tag_idx_sents = scores_sents.map(
                lambda x: tf.argmax(x, axis=-1, output_type=tf.int32)
            )
            tag_idx_sents = [s.numpy() for s in tag_idx_sents]
        return tag_idx_sents

    # args and kwargs are the same as original predict function
    # https://www.tensorflow.org/api_docs/python/tf/keras/Model
    def predict_id(self, x_batches, **kwargs):
        """

        :param x_batches: all data in multiple batches
        :param kwargs:
        :return:
        """

        tag_idx_sents = self.predict_idx(x_batches, **kwargs)
        tag_ids_sents = list(map(
            lambda x: self.all_tag_ids[x],
            tag_idx_sents
        ))
        return tag_ids_sents

    def predict_label(self, x_batches, **kwargs):
        """

        :param x_batches: all data in multiple batches
        :param kwargs:
        :return:
        """
        tag_idx_sents = self.predict_idx(x_batches, **kwargs)
        label_predictions = list(map(
            lambda x: self.all_tags[x],
            tag_idx_sents
        ))
        if self.clean_tag:
            label_predictions = list(map(self.remove_broken_entities, label_predictions))
        return label_predictions

    def remove_broken_entities(self, tag_seq):
        clean_seq = []
        for tag in tag_seq:
            if (
                tag.startswith('B-')
                or tag.startswith('S-')
                or tag.startswith('O')
            ):
                clean_seq.append(tag)
            elif (
                len(clean_seq) > 0
                and clean_seq[-1][2:] == tag[2:]
                and (not clean_seq[-1].startswith('E-'))
            ):
                clean_seq.append(tag)
            else:
                clean_seq.append('O')
        return clean_seq

    def evaluate_id(self, x_batches, y_true, raw_sentences, **kwargs):
        """

        :param x_batches: all data in multiple batches
        :param y_true:
        :param raw_sentences:
        :param kwargs:
        :return: averaged F1 score
        """

        labels_true = y_true.map(
            lambda x: self.tag_id_to_tag_lookup.lookup(x)
        )

        return self.evaluate_label(x_batches, labels_true, raw_sentences, **kwargs)

    def evaluate_label(self, x_batches, labels_true, raw_sentences, **kwargs):
        tags_pred = self.predict_label(x_batches)

        # labels_true was padded, here we recover it
        labels_true = labels_true.unbatch()
        word_mask = x_batches.map(
            lambda x: tf.cast(x['score_mask'], tf.bool)
        ).unbatch()
        labels_mask = tf.data.Dataset.zip((labels_true, word_mask))
        # restore the length from max_seq_len to real_seq_len
        labels_true = labels_mask.map(lambda l, mask: l[mask])
        labels_true = [l.numpy().astype('U') for l in labels_true]

        predictions = []
        count = np.zeros((self.n_tags, self.n_tags), dtype=np.int32)

        for sent_index, raw_sentence in enumerate(raw_sentences):
            r_tags = labels_true[sent_index]
            r_idx = [self.tag_tag_to_idx[t] for t in r_tags]

            p_tags = tags_pred[sent_index]
            p_idx = [self.tag_tag_to_idx[t] for t in p_tags]

            assert len(p_tags) == len(r_tags)
            assert len(r_tags) == len(raw_sentence)

            if self.tag_scheme == 'iobes':
                p_tags = iobes_iob(p_tags)
                r_tags = iobes_iob(r_tags)
            for i, (i_pred, i_real) in enumerate(zip(p_idx, r_idx)):
                new_line = " ".join(
                    [raw_sentence[i]['text'],  r_tags[i], p_tags[i]]
                )
                predictions.append(new_line)
                count[i_real, i_pred] += 1
            predictions.append("")

        eval_lines = evaluate_lines(predictions)
        eval_lines = eval_lines.rstrip()
        eval_lines = eval_lines.split('\n')
        eval_lines = [l.rstrip() for l in eval_lines]
        for line in eval_lines:
            print(line)

        # Confusion matrix with accuracy for each tag
        print(("{: >2}{: >7}{: >7}%s{: >9}" % ("{: >7}" * self.n_tags)).format(
            "ID", "NE", "Total",
            *([tag for tag in self.all_tags] + ["Percent"])
        ))
        for i in range(self.n_tags):
            print(("{: >2}{: >7}{: >7}%s{: >9}" % ("{: >7}" * self.n_tags)).format(
                str(i), self.all_tags[i], str(count[i].sum()),
                *([count[i][j] for j in range(self.n_tags)] +
                  ["%.3f" % (count[i][i] * 100. / max(1, count[i].sum()))])
            ))

        # Global accuracy
        print("%i/%i (%.5f%%)" % (
            count.trace(), count.sum(),
            100. * count.trace() / max(1, count.sum())
        ))

        return float(eval_lines[1].strip().split()[-1])

    def save_model(self, save_weights=False):
        """
        Write components values to disk.
        """

        signature = inspect.signature(self.__class__.__init__)
        model_config_essential = dict(filter(
            lambda x: x[0] in signature.parameters,
            self.__dict__.items()
        ))
        with open(
            os.path.join(self.model_path, 'model_config.pkl'),
            'wb'
        ) as fw:
            pickle.dump(model_config_essential, fw)
        if save_weights:
            self.save_weights(self.opt_cp_path)

    @classmethod
    def create_scratch_model(cls, model_path, bert_path=None, to_reload_model=True):
        """
        Create model from config file w/o initialization

        :param model_path:
        :return:
        """
        config_path = os.path.join(model_path, 'model_config.pkl')
        with open(config_path, 'rb') as fr:
            model_config = pickle.load(fr)
        model_config['model_path'] = model_path
        if bert_path:
            model_config['bert_path'] = bert_path
        model_config['to_reload_model'] = to_reload_model
        model = cls(**model_config)
        batch_size = model_config['batch_size']
        # run on one sample to build model
        data_dicts = [
            {
                'words': [1, ],
                'chars': [[1,], ],
                'ele_num': [0.0, ],
                'only_CHO': [0, ],
                'tar_tag': [0, ],
                'pre_tag': [0, ],
                'tags': [1, ],
                'length': 1,
                'num_tokens': 1,
                'bert_token_type_ids': [0, ],
                'bert_attention_mask': [1, ],
                'score_mask': [1, ],
            }
        ] * batch_size
        data_type, data_shape, padded_data_shape = get_input_format(model_type='MER')
        data_X, data_Y = dict_to_tf_dataset(
            data_dicts,
            data_type,
            data_shape,
            padded_shape=padded_data_shape,
            column_y=None,
            batch_size=batch_size,
        )
        # the weight has been changed even only fit once
        # need to reload weight to use it
        # TODO: is there anyway to compile the model without calling fit?
        model.fit(
            x=tf.data.Dataset.zip((data_X, data_Y)),
            epochs=1,
            verbose=0,
        )
        return model

    @classmethod
    def reload_model(cls, model_path, cp_path=None, bert_path=None):
        """
        Load components values from disk.
        """
        # TODO: should be able to create scratch bert model with vocab
        #  rather than whole checkpoint file
        model = NERModel.create_scratch_model(
            model_path=model_path,
            bert_path=bert_path,
            to_reload_model=True,
        )
        if cp_path is None:
            cp_path = os.path.join(model_path, 'opt_cp', 'cp.ckpt')
        model.load_weights(cp_path)
        return model


"""
End to end test of video timelines.
"""

import datetime
import logging

from edx.analytics.tasks.tests.acceptance import AcceptanceTestCase

log = logging.getLogger(__name__)

class VideoAcceptanceTest(AcceptanceTestCase):
    """End to end test of video timelines."""

    INPUT_FILE = 'video_timeline.log'

    def test_video_timeline(self):
        self.upload_tracking_log(self.INPUT_FILE, datetime.date(2014, 5, 2))

        self.task.launch([
            'InsertToMysqlAllVideoTask',
            '--interval', '2014-05-01-2014-05-06',
            '--n-reduce-tasks', str(self.NUM_REDUCERS),
        ])

        self.validate_video_info()
        self.validate_video_timeline()

    def validate_video_info(self):
        """Ensure the video information is correct."""
        with self.export_db.cursor() as cursor:
            cursor.execute(
                'SELECT pipeline_video_id, course_id, encoded_module_id, duration, segment_length,'
                ' users_at_start, users_at_end, total_viewed_seconds FROM video'
                ' ORDER BY pipeline_video_id ASC'
            )
            results = cursor.fetchall()

        expected = [
            (
                'course-v1:edX+DemoX+Test_2014|0b9e39477cf34507a7a48f74be381fdd',
                'course-v1:edX+DemoX+Test_2014',
                '0b9e39477cf34507a7a48f74be381fdd',
                4,
                5,
                1,
                1,
                5,
            ),
            (
                'edX/DemoX/Demo_Course|i4x-edX-DemoX-video-3cb54a11efae4ccc8a0aade24d14b255',
                'edX/DemoX/Demo_Course',
                'i4x-edX-DemoX-video-3cb54a11efae4ccc8a0aade24d14b255',
                24,
                5,
                2,
                2,
                60,
            ),
            (
                'edX/DemoX/Demo_Course|i4x-edX-DemoX-video-8c0028eb2a724f48a074bc184cd8635f',
                'edX/DemoX/Demo_Course',
                'i4x-edX-DemoX-video-8c0028eb2a724f48a074bc184cd8635f',
                24,
                5,
                0,
                1,
                15,
            ),
            (
                'course-v1:edX+DemoX+Test_2015|0b9e39477cf34507a7a48f74be381fdd1',
                'course-v1:edX+DemoX+Test_2015',
                '0b9e39477cf34507a7a48f74be381fdd1',
                444,
                5,
                1,
                0,
                5
            ),
        ]
        self.assertItemsEqual(expected, results)

    def validate_video_timeline(self):
        """Ensure the video timeline counts are correct."""
        with self.export_db.cursor() as cursor:
            cursor.execute(
                'SELECT pipeline_video_id, segment, num_users, num_views FROM video_timeline'
                ' ORDER BY pipeline_video_id, segment ASC'
            )
            results = cursor.fetchall()

        expected = [
            ('course-v1:edX+DemoX+Test_2014|0b9e39477cf34507a7a48f74be381fdd', 0, 1, 1),
            ('edX/DemoX/Demo_Course|i4x-edX-DemoX-video-3cb54a11efae4ccc8a0aade24d14b255', 0, 2, 2),
            ('edX/DemoX/Demo_Course|i4x-edX-DemoX-video-3cb54a11efae4ccc8a0aade24d14b255', 1, 2, 3),
            ('edX/DemoX/Demo_Course|i4x-edX-DemoX-video-3cb54a11efae4ccc8a0aade24d14b255', 2, 1, 2),
            ('edX/DemoX/Demo_Course|i4x-edX-DemoX-video-3cb54a11efae4ccc8a0aade24d14b255', 3, 2, 2),
            ('edX/DemoX/Demo_Course|i4x-edX-DemoX-video-3cb54a11efae4ccc8a0aade24d14b255', 4, 2, 3),
            ('edX/DemoX/Demo_Course|i4x-edX-DemoX-video-8c0028eb2a724f48a074bc184cd8635f', 2, 1, 1),
            ('edX/DemoX/Demo_Course|i4x-edX-DemoX-video-8c0028eb2a724f48a074bc184cd8635f', 3, 1, 1),
            ('edX/DemoX/Demo_Course|i4x-edX-DemoX-video-8c0028eb2a724f48a074bc184cd8635f', 4, 1, 1),
            ('course-v1:edX+DemoX+Test_2015|0b9e39477cf34507a7a48f74be381fdd1', 0, 1, 1),
        ]
        self.assertItemsEqual(expected, results)


from math import inf
from collections import deque
from tracemalloc import start

def dijkstra(wmat, start, end=-1):
    n = len(wmat)

    dist = [inf] * n
    dist[start] = wmat[start][start]  # 0

    spVertex = [False] * n
    parent = [-1] * n

    path = [{}] * n

    for count in range(n - 1):
        minix = inf
        u = 0

        for v in range(len(spVertex)):
            if spVertex[v] == False and dist[v] <= minix:
                minix = dist[v]
                u = v

        spVertex[u] = True
        for v in range(n):
            if not (spVertex[v]) and wmat[u][v] != 0 and dist[u] + wmat[u][v] < dist[v]:
                parent[v] = u
                dist[v] = dist[u] + wmat[u][v]

    for i in range(n):
        j = i
        s = []
        while parent[j] != -1:
            s.append(j)
            j = parent[j]
        s.append(start)
        path[i] = s[::-1]

    return (dist[end], path[end]) if end >= 0 else (dist, path)

def checkbipartite(graph):
    N = len(graph)
    col = [-1] * N
    for i in range(N):
        if col[i] != -1:
            continue
        q = deque()
        q.append((0, 0))
        # bfs
        while q:
            node, color = q.popleft()
            if col[node] == -1:
                col[node] = color
                for nx in graph[node]:
                    q.append((nx, color ^ 1))
        if col[node] != color:
            return False
    return True

def getdegree(graph):
    for ind in range(len(graph)):
        count = 0
        x = graph[ind]
        for i in x:
            if i > 0:
                count += 1
        print("degree of " + str(ind) + " is " + str(count))

def isConnected(graph, V):
    # Mark all the vertices as not visited
    visited = [False] * (V)

    #  Find a vertex with non-zero degree
    for i in range(V):
        if len(graph[i]) > 1:
            break

    # If there are no edges in the graph, return true
    if i == V - 1:
        return True

    # Start DFS traversal from a vertex with non-zero degree
    DFSUtil(graph, i, visited)

    # Check if all non-zero degree vertices are visited
    for i in range(V):
        if visited[i] == False and len(graph[i]) > 0:
            return False

    return True

def DFSUtil(graph, v, visited):
    # Mark the current node as visited
    visited[v] = True

    # Recur for all the vertices adjacent to this vertex
    for i in graph[v]:
        if visited[i] == False:
            DFSUtil(graph, i, visited)

def isEulerian(graph, V):
    # Check if all non-zero degree vertices are connected
    if isConnected(graph, V) == False:
        return 0
    else:
        # Count vertices with odd degree
        odd = 0
        for i in range(V):
            if len(graph[i]) % 2 != 0:
                odd += 1

        '''If odd count is 2, then semi-eulerian.
        If odd count is 0, then eulerian
        If count is more than 2, then graph is not Eulerian
        Note that odd count can never be 1 for undirected graph'''
        if odd == 0:
            return 2
        elif odd == 2:
            return 1
        elif odd > 2:
            return 0

def testeuler(graph):
    res = isEulerian(graph, len(graph))
    if res == 0:
        print("graph is not Eulerian")
    elif res == 1:
        print("graph has a Euler path")
    else:
        print("graph has a Euler cycle")

def findpaths(graph, start, end):
    def dfs(node, path, output):
        if node == end:
            output.append(path)
        for nx in range(len(graph[node])):
            if nx not in path:
                if graph[node][nx] > 0:
                    dfs(nx, path + [nx], output)

    output = []
    dfs(start, [start], output)
    return output

testgraph = [[0, 2, 0, 0, 0, 1, 0, 0],
             [2, 0, 2, 2, 4, 0, 0, 0],
             [0, 2, 0, 0, 3, 0, 0, 1],
             [0, 2, 0, 0, 4, 3, 0, 0],
             [0, 4, 3, 4, 0, 0, 7, 0],
             [1, 0, 0, 3, 0, 0, 5, 0],
             [0, 0, 0, 0, 7, 5, 0, 6],
             [0, 0, 1, 0, 0, 0, 6, 0]]
start = 0
end = 3

print("==>> paths")
print(findpaths(testgraph, start, end))
print("==>> check euler")
testeuler(testgraph)
print("==>> check degree")
getdegree(testgraph)
print("==>> isbiparite " + str(checkbipartite(testgraph)))
print("==>> dijkstra ", dijkstra(testgraph, start, end)[1])


import PyPDF2

#translate the pdf file to txt file
def pdf_to_text(pdf_file_path, txt_file_path):
    try:
        # Open the PDF file
        with open(pdf_file_path, 'rb') as pdf_file:
            pdf_reader = PyPDF2.PdfReader(pdf_file)
            # Initialize an empty text string
            text = ""

            # Loop through each page and extract text
            for page_num in range(len(pdf_reader.pages)):
                page = pdf_reader.pages[page_num]
                text += page.extract_text()

            # Save the extracted text to a text file
            #creates new text file if text file doesnt exist
            #overwrites content in text file if text file does exist
            with open(txt_file_path, 'w', encoding='utf-8') as txt_file:
                txt_file.write(text)

        print(f"Text extracted from '{pdf_file_path}' and saved to '{txt_file_path}'.")

    except FileNotFoundError:
        print(f"The file '{pdf_file_path}' was not found.")
    except Exception as e:
        print(f"An error occurred: {str(e)}")


import graphene
from graphene import Field

from graphene_django.types import ErrorType
from graphene.types.base import BaseOptions

from django.core.exceptions import ObjectDoesNotExist

class DeleteMutationOptions(BaseOptions):
    model = None
    arguments = None
    output = None
    resolver = None
    interfaces = ()
    fields = None

class DeleteMutation(graphene.Mutation):
    class Meta:
        abstract = True
    id = graphene.ID()
    ok = graphene.Boolean(
        description="Boolean field that return mutation result request."
    )
    errors = graphene.List(ErrorType, description="Errors list for the field")

    class Arguments:
        id = graphene.ID(required=True)

    @classmethod
    def __init_subclass_with_meta__(
        cls,
        model=None,
        interfaces=(),
        resolver=None,
        output=None,
        arguments=None,
        _meta=None,
        **options
    ):
        if not _meta:
            _meta = DeleteMutationOptions(cls)

        assert model is not None, ("Model class in {} must be need.").format(
            cls.__name__
        )
        _meta.model = model
        super(DeleteMutation, cls).__init_subclass_with_meta__(
            interfaces=interfaces,
            resolver=resolver,
            output=output,
            arguments=arguments,
            _meta=_meta,
            **options
        )

    @classmethod
    def get_errors(cls, errors):
        errors_dict = {"ok": False, "errors": errors, "id": None}

        return cls(**errors_dict)

    @classmethod
    def perform_mutate(cls, obj, info):
        resp = {"ok": True, "errors": None, "id": obj.id}

        return cls(**resp)

    @classmethod
    def mutate(cls, root, info, id):
        try:
            obj = cls._meta.model.objects.get(id=id)
            obj.delete()
            obj.id = id
            return cls.perform_mutate(obj, info)
        except ObjectDoesNotExist:
            return cls.get_errors(
                [
                    ErrorType(
                        field="id",
                        messages=[
                            "A {} obj with id {} do not exist".format(
                                cls._meta.model.__name__, id
                            )
                        ],
                    )
                ]
            )

    @classmethod
    def Field(
        cls, name=None, description=None, deprecation_reason=None, required=False
    ):
        """ Mount instance of mutation Field. """
        return Field(
            cls._meta.output,
            args=cls._meta.arguments,
            resolver=cls._meta.resolver,
            name=name,
            description=description or cls._meta.description,
            deprecation_reason=deprecation_reason,
            required=required,
        )


from flask import Flask, render_template , Response
import cv2 , os
import argparse

app = Flask(__name__)
app.config['SECRET_KEY'] = '1234'

@app.route('/') #html 보여준다.
def index():
    return render_template('index.html')

@app.route('/video_show')
def video_show():
    return Response(yield_video(), mimetype= "multipart/x-mixed-replace; boundary=frame")

def yield_video():
    global video
    if video == 'default':
        capture = cv2.VideoCapture(0)
    else:
        path = 'videosrc/'+ video # video 파일경로 접근
        if os.path.isfile(path) == False:
            capture = cv2.VideoCapture(0)
        else:
            capture = cv2.VideoCapture(path)

    encode_param = [int(cv2.IMWRITE_JPEG_QUALITY), 90] #변환 퀄리티 90프로로 지정

    while True:
        ret, frame = capture.read()
        if ret != True:
            break
        frame = cv2.resize(frame,dsize=(0,0),fx=0.6,fy=0.4,interpolation=cv2.INTER_LANCZOS4 )
        frame = cv2.imencode('.jpg',frame, encode_param)[1].tobytes()
        yield(b'--frame\r\n' b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n')

        if cv2.waitKey(1)>0 : break

    print("Video Done")
if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Echo websocketClient --movie MOVIE")
    parser.add_argument('--movie', help = "video-directory", default= 'default')
    args = parser.parse_args()
    video = args.movie
    app.run()
#pexels-ekaterina-bolovtsova-7003250.mp4
#production_ID_5077814.mp4


""" HSC Datasets """

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from . import hsc_utils
from . import astroimage_utils

from tensor2tensor.data_generators import generator_utils
from tensor2tensor.data_generators import image_utils
from tensor2tensor.data_generators import problem
from tensor2tensor.layers import modalities, common_layers
from tensor2tensor.utils import metrics

from galaxy2galaxy.utils import registry

from scipy.ndimage import gaussian_filter

import tensorflow as tf
import numpy as np
import fits2hdf.pyhdfits as fits
from astropy.table import Table
from astropy.visualization import make_lupton_rgb
import h5py
import os

# HSC default pixel scale
_HSC_PIXEL_SCALE=0.168 #arcsec
# Path to sql files for HSC samples
_HSC_SAMPLE_SQL_DIR=os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data')

def _resize_image(im, size):
  centh = im.shape[0]/2
  centw = im.shape[1]/2
  lh, rh = int(centh-size/2), int(centh+size/2)
  lw, rw = int(centw-size/2), int(centw+size/2)
  cropped = im[lh:rh, lw:rw, :]
  assert cropped.shape[0]==size and cropped.shape[1]==size, f"Wrong size! Still {cropped.shape}"
  return cropped

@registry.register_problem
class Img2imgHSC(astroimage_utils.AstroImageProblem):
  """Base class for image problems created from HSC Public Data Release.
  """

  # START: Subclass interface
  def hparams(self, defaults, model_hparams):
    p = defaults
    p.img_len = 64
    p.filters = ['HSC-G', 'HSC-R', 'HSC-I', 'HSC-Z']
    p.sql_file = os.path.join(_HSC_SAMPLE_SQL_DIR, 'hsc_pdr2_wide_anomaly_small.sql')
    p.data_release = 'pdr2'
    p.rerun = 'pdr2_wide'

  @property
  def num_bands(self):
    """Number of bands."""
    p = self.get_hparams()
    return len(p.filters)

  def generator(self, data_dir, tmp_dir, dataset_split, task_id=-1):
    """ Generator yielding individual postage stamps.
    """
    p = self.get_hparams()

    # Step 1: Maybe regenerate the dataset
    self.maybe_build_dataset(tmp_dir)
    import timeit

    # Step 2: Extract postage stamps, resize them to requested size
    catalog = Table.read(os.path.join(tmp_dir, 'catalog.fits'))
    with h5py.File(os.path.join(tmp_dir, 'cutouts.hdf'),'r') as cutouts:
        # Loop through the examples, resize cutout to desired size
        for row in catalog:
          cutout = cutouts[str(row['object_id'])]
          im = [cutout[f]['HDU0']['DATA'][:] for f in p.filters]

          try:
              im = np.stack(im, axis=-1).astype('float32')
          except:
              print('Failure to stack bands', [i.shape for i in im])
              continue

          # Images may not have exactly the right number of pixels
          im = _resize_image(im, p.img_len)

          serialized_output = {"image/encoded": [im.tostring()],
                               "image/format": ["raw"]}

          # If attributes are requested, let's add them to the dataset
          if hasattr(p, 'attributes'):
            for k in p.attributes:
              serialized_output['attrs/'+k] = [np.asscalar(row[k])]

          yield serialized_output

  def preprocess_example(self, example, unused_mode, unused_hparams):
    """ Preprocess the examples, can be used for further augmentation or
    image standardization.
    """
    p = self.get_hparams()
    image = example["inputs"]

    # TODO: apply some pre-processing/normalization to the images

    if hasattr(p, 'attributes'):
      example["attributes"] = tf.stack([example[k] for k in p.attributes])

    example["inputs"] = image
    example["targets"] = image
    return example

  # END: Subclass interface
  @property
  def is_generate_per_split(self):
    return False

  def maybe_build_dataset(self, tmp_dir):
    """ Check that the HSC dataset is downloaded and ready in tmp_dir, otherwise
    redownload it from the server.
    """
    p = self.get_hparams()
    if (not os.path.isfile(os.path.join(tmp_dir, 'catalog.fits')) or
        not os.path.isfile(os.path.join(tmp_dir, 'cutouts.hdf')) ):
      hsc_utils.build_hsc_sample(p.sql_file,
                                 out_dir=tmp_dir,
                                 tmp_dir=os.path.join(tmp_dir,'tmp'),
                                 cutout_size=(1.5+p.img_len)*_HSC_PIXEL_SCALE/2, # Requesting slightly larger pixel size to avoid roundoff errors
                                 filters=p.filters,
                                 data_release=p.data_release,
                                 rerun=p.rerun)

@registry.register_problem
class Img2imgHSCAnomaly(Img2imgHSC):
  """ Dataset for anomaly detection on HSC data.
  """

  def hparams(self, defaults, model_hparams):
    p = defaults
    p.img_len = 128
    p.filters = ['HSC-G', 'HSC-R', 'HSC-I']
    p.sql_file = os.path.join(_HSC_SAMPLE_SQL_DIR, 'hsc_pdr2_wide_anomaly.sql')
    p.data_release = 'pdr2'
    p.rerun = 'pdr2_wide'
    p.attributes = ['g_cmodel_mag', 'r_cmodel_mag', 'i_cmodel_mag', 'z_cmodel_mag']
    p.modality = {"inputs": modalities.ModalityType.IDENTITY,
                  "attributes":  modalities.ModalityType.IDENTITY,
                  "targets": modalities.ModalityType.IDENTITY}
    p.vocab_size = {"inputs": None,
                    "attributes": None,
                    "targets": None}

  def preprocess_example(self, example, unused_mode, unused_hparams):
    """ Luptonize the examples, so that we can use t2t models easily
    """
    p = self.get_hparams()
    image = example["inputs"]

    # Apply Luptonic Asinh stretch, and return uint8 rgb images
    def my_func(x):
      return make_lupton_rgb(x[...,2], x[...,1], x[...,0], Q=15, stretch=0.5, minimum=0)

    int_image = tf.py_func(my_func, [image], tf.uint8)
    int_image.set_shape(image.shape)
    image = common_layers.convert_rgb_to_symmetric_real(int_image)

    if hasattr(p, 'attributes'):
      example["attributes"] = tf.stack([example[k] for k in p.attributes])

    example["inputs"] = image
    example["targets"] = image
    return example

@registry.register_problem
class Img2imgHSCAnomalySmall(Img2imgHSCAnomaly):
  """ Dataset for anomaly detection on HSC data, but smaller.
  """

  def hparams(self, defaults, model_hparams):
    p = defaults
    p.img_len = 128
    p.filters = ['HSC-G', 'HSC-R', 'HSC-I']
    p.sql_file = os.path.join(_HSC_SAMPLE_SQL_DIR, 'hsc_pdr2_wide_anomaly_small.sql')
    p.data_release = 'pdr2'
    p.rerun = 'pdr2_wide'
    p.attributes = ['g_cmodel_mag', 'r_cmodel_mag', 'i_cmodel_mag', 'z_cmodel_mag']
    p.modality = {"inputs": modalities.ModalityType.IDENTITY,
                  "attributes":  modalities.ModalityType.IDENTITY,
                  "targets": modalities.ModalityType.IDENTITY}
    p.vocab_size = {"inputs": None,
                    "attributes": None,
                    "targets": None}

@registry.register_problem
class Img2photozHSC(Img2imgHSC):
  """ Dataset for photoz estimation on HSC data.
  """

  def hparams(self, defaults, model_hparams):
    p = defaults
    p.img_len = 64
    p.filters = ['HSC-G', 'HSC-R', 'HSC-I', 'HSC-Z', 'HSC-Y']
    p.sql_file = os.path.join(_HSC_SAMPLE_SQL_DIR, 'hsc_pdr2_wide_photoz.sql')
    p.data_release = 'pdr2'
    p.rerun = 'pdr2_wide'
    p.attributes = ['a_g', 'a_r', 'a_i', 'a_z', 'a_y',
                   'g_cmodel_mag', 'r_cmodel_mag', 'i_cmodel_mag', 'z_cmodel_mag',
                   'specz_redshift', 'specz_redshift_err']
    p.modality = {"inputs": modalities.ModalityType.IDENTITY,
                  "attributes":  modalities.ModalityType.IDENTITY,
                  "targets": modalities.ModalityType.IDENTITY}
    p.vocab_size = {"inputs": None,
                    "attributes": None,
                    "targets": None}

  def preprocess_example(self, example, unused_mode, unused_hparams):
    """ Luptonize the examples, so that we can use t2t models easily
    """
    p = self.get_hparams()
    image = example["inputs"]

    if hasattr(p, 'attributes'):
      example["attributes"] = tf.stack([example[k] for k in p.attributes])

    example["inputs"] = image
    example["targets"] = example['specz_redshift']
    return example


from kivy.uix.screenmanager import Screen
from kivy.lang import Builder
from kivy.uix.boxlayout import BoxLayout
from kivy.properties import StringProperty
from kivy.utils import platform

from kivymd.uix.dialog import MDDialog
from kivymd.uix.button import MDFlatButton
from kivymd.uix.filemanager import MDFileManager

import socket
import pathlib
from struct import pack

Builder.load_string("""
<ReceiverPropmtContent>:
    orientation: 'vertical'
    spacing: "12dp"
    size_hint_y: None
    height: "50dp"
    MDTextField:
        id: receiver_ip
        hint_text: "Receiver IP"
        helper_text: "Find the ip of the reciever in the reciever's screen"
        helper_text_mode: 'on_focus'

<SendScreen>:
    id: send_screen
    name: 'send_screen'
    MDRaisedButton:
        text: "Select File to Send"
        elevation: 10
        pos_hint: {'center_x': 0.5, 'center_y': 0.5}
        on_release: root.open_filemanager()
""")

class Sender:
    address = None
    port = 5001

    socket = None

    def __init__(self, address):
        self.address = address

    def connect(self):
        self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.socket.connect((self.address, self.port))
        return self.socket

    def disconnect(self):
        self.socket.close()

    def convert_int(self, integer):
        return pack('!Q', integer)

class ReceiverPropmtContent(BoxLayout):
    pass

class SendScreen(Screen):
    receiver_ip = StringProperty("")

    def __init__(self, *args, **kwargs):
        super(SendScreen, self).__init__(*args, **kwargs)

        self.file_manager = MDFileManager(
            exit_manager=self.on_filemanager_exit,
            select_path=self.on_select_path,
            multiselect=True
        )

    def send_file(self, paths):

        # SEND NUMBER OF FILES
        self.socket.sendall(self.sender.convert_int(len(paths)))

        for path in paths:
            path_obj = pathlib.Path(path)
            file_name = path_obj.name
            file_size = path_obj.stat().st_size

            # SEND FILE NAME SIZE
            self.socket.sendall(self.sender.convert_int(len(file_name)))
            # SEND FILE NAME
            self.socket.sendall(file_name.encode('utf-8'))

            # SEND FILE SIZE
            self.socket.sendall(self.sender.convert_int(file_size))

            # SEND FILE
            with open(path, 'rb') as file:
                self.socket.sendfile(file)

            print("SENT !")

        self.sender.disconnect()

    def on_filemanager_exit(self, *args):
        self.file_manager.close()

    def on_select_path(self, paths):
        self.send_file(paths)
        self.file_manager.close()

    def open_filemanager(self, *args):
        path = "/"
        if platform == 'android':
            path = '/storage/emulated/0/'
        self.file_manager.show(path)

    def set_receiver_ip(self, value):
        self.receiver_ip = value

        # CONNECT TO RECEIVER
        self.sender = Sender(self.receiver_ip)
        self.socket = self.sender.connect()

        self.receiver_prompt.dismiss()

    def on_enter(self):
        self.receiver_prompt = MDDialog(
            title="Enter Receiver IP",
            type="custom",
            auto_dismiss=False,
            content_cls=ReceiverPropmtContent(),
            buttons=[
                MDFlatButton(
                    text="OK",
                    on_release=lambda x: self.set_receiver_ip(
                        self.receiver_prompt.content_cls.ids.receiver_ip.text
                    )
                )
            ]
        )
        self.receiver_prompt.open()


# -*- coding: utf-8 -*-
"""
Created on Fri Oct 29 18:16:02 2021

@author: Park Ihn
"""
import json
from collections import OrderedDict
import csv
from datetime import datetime
import re
from turtle import Turtle
#input file open
inputCsv = open("KIKmix.20220401(말소코드포함).csv","r",encoding="cp949")
rd = csv.reader(inputCsv)
lines = list(rd) #list from csv file lines

### GUI 형식으로
### 파일을 선택하면 입력됨
###

city=''
gu=''

del lines[0]
## 첫행 삭제

bIsSiUpdate = False
bIsGuUpdate = False

fileList = []
tempDong = []
si_check = []
MeetingRecord = OrderedDict()
MeetingRecord["city"] = city
MeetingRecord["gu"] = gu
MeetingRecord["dong"] = tempDong

for l in lines:

    if(l[5]==l[1]):
        gu_check = []
        bIsSiUpdate= True
    if(l[5]==l[2]):
        gu_check.append(gu)
        check = True
        for gu_list in gu_check:
            if (gu_list == l[5]):
                    check = False
        if(check):
            bIsGuUpdate = True
            gu = l[5]

    if(bIsSiUpdate):
        bIsSiUpdate= False
        city = l[5] #시 값 update
        continue
    else:
        if(bIsGuUpdate): #구의 값이 행정동과 법정동이 같을 때
            tempDong=[]
            MeetingRecord = OrderedDict() #순서대로 넣어라
            MeetingRecord["city"] = city #시의 값을 넣고
            MeetingRecord["gu"] = gu #구의 값을 넣고
            MeetingRecord["dong"] = tempDong
            fileList.append(MeetingRecord)
            #print(fileList)
            bIsGuUpdate = False
        else:
            if((l[3]!='') and (l[3] not in tempDong) ): # 정규식 추가하기
                #동의 값이 빈칸이 아니고, 동의 값이 tempdong에 없을 때
                m = re.search('제\d', l[3])
                if m == None:
                # 문자열 '제1'의 값이 매치되는 값이 없다면 동의 값을 그대로 삽입
                   tempDong.append(l[3])
                elif m != None:
                   n = m.start()
                   if n >= 1:
                # 문자열 '제1'의 값이 있고, 첫글자가 '제' 아닐 때
                    mm =     re.search('제제',l[3])
                   if (mm == None):
                       ## '거제제1동'같은 문자열이 없다면,
                      k= re.sub(r'제', "", l[3])
                      tempDong.append(l[3])
                      ## '묵제1동' 삽입
                      tempDong.append(k)
                      ## '묵1동' 삽입
                   else:
                      ## '게제제1동과 같은 문자열이 있다면, 거제1동으로 삽입
                      k= re.sub(r'제제', "제", l[3])
                      tempDong.append(l[3])
                      tempDong.append(k)
                   #  '거제제1동' 삽입
                   #  '거제1동' 삽입
                   #  첫글자가 '제'가 아닌 문자열에서 '제'를 삭제한 '묵동1동'을 삽입
                   # '묵동제1동'과 '묵동1동'의 값을 동시에 추가
            if(l[5] != ""):
                if(l[5] not in tempDong):
                    tempDong.append(l[5])

MeetingRecord = OrderedDict()
MeetingRecord["city"] = city
MeetingRecord["gu"] = gu
MeetingRecord["dong"] = tempDong
fileList.append(MeetingRecord)
tempDong=[]

now = datetime.now()
with open("output"+str(now.month)+str(now.day)+str(now.hour)+str(now.minute)+"_법정동행정동모두.json",'w',encoding = 'utf-8') as f:
    json.dump(fileList, f, ensure_ascii=False, indent="\t")

print("Complete\n");
inputCsv.close()


import socket
import select
import errno
from select import POLLIN, POLLOUT, POLLERR, POLLHUP, POLLNVAL
from ccstruct import *
from collections import deque
# define channel types
CH_TYPE_PIPE  = 0
CH_TYPE_TCP   = 1
CH_TYPE_UDP_S = 2  # send-only UDP
CH_TYPE_UDP_R = 3  # recv-only UDP
# define channel state
CH_STATE_NOTCONN = 0
CH_STATE_PRECONN = 1     # to handle EINPROGRESS after non-blocking connect()
CH_STATE_CONNECT = 2     # TCP connected
CH_STATE_LISTEN = 3      # listening
CH_STATE_CLOSE = 4       # Channel will be closed
# define channel eventmask
CH_READ  = POLLIN
CH_WRITE = POLLOUT
CH_ERROR = POLLERR

class Buffer:
    """ Message buffer
        Each buffer contains the serialized wait-to-send/wait-to-receive ccfd packet
    """
    def __init__(self, data=None):
        if not data:
            self.data = bytearray()   #message itself
            self.length = 0       # length of message to receive
        else:
            self.data = data
            self.length = len(data)
        self.pos = 0       # Position of read/write

class MsgQueue:
    def __init__(self):
        self.messages = deque()

    def enqueue(self, buf):
        self.messages.append(buf)

    def dequeue(self):
        return self.messages.popleft()

    def size(self):
        return len(self.messages)

    def is_empty(self):
        return (not self.size())

    def first(self):
        """ Return the very first message in the queue
        """
        if self.is_empty():
            return None
        else:
            return self.messages[0]

    def last(self):
        """ Return most recent message being worked on
        """
        if self.is_empty():
            return None
        else:
            return self.messages[self.size()-1]

class Channel:
    """ Each point-to-point communication is performed via a channel
    """
    def __init__(self, handle=-1, chan_t=CH_TYPE_TCP, remote=None):
        self.chid   = -1
        self.handle = handle    # handle of the channel (fd)
        self.chan_t = chan_t    # channel type
        self.remote = remote    # remote address tuple (ip, port)
        self.state = CH_STATE_NOTCONN
        self.sendq = MsgQueue()    # send queue (first-in-first-out)
        self.recvq = MsgQueue()    # recv queue (first-in-first-out)
        self.eventmask = 0      # events on the handle

    def set_channel_id(self, id):
        self.chid = id

    def send(self, data):
        self.sendq.enqueue(Buffer(data))
        return

    def receive(self):
        if not self.recvq.is_empty() and \
                self.recvq.first().length == self.recvq.first().pos:
            return self.recvq.dequeue().data
        else:
            self.eventmask &= CH_READ
            return None

    def doread(self):
        """ read from the handle and store messages to recvq
        """
        if self.recvq.is_empty():
            self.recvq.enqueue(Buffer())

        # Case 1) Read from pipe
        if self.chan_t == CH_TYPE_PIPE:
            data = self.handle.recv()
            self.recvq.last().data += data
            self.recvq.last().length = len(data)
            self.recvq.last().pos += len(data)
            self.eventmask |= CH_READ
            return

        # Case 2) Try to read from socket
        # Start with header
        if self.recvq.last().length == 0:
            self.recvq.last().length = hdrlen()
            self.recvq.last().pos = 0
        # A complete message is here, set ready to process.
        if self.recvq.last().length == self.recvq.last().pos:
            self.eventmask |= CH_READ
            return

        try:
            data, addr = self.handle.recvfrom(self.recvq.last().length-self.recvq.last().pos,
                                                socket.MSG_DONTWAIT)
        except Exception as details:
            print("Channel.doread() cannot recvfrom(). Error: %s" % (details, ))
            self.eventmask = CH_ERROR
            return
        if len(data) == 0:
            print("Channel.doread() didn't read anything, something is wrong")
            self.eventmask = CH_ERROR
            return
        self.recvq.last().data += data
        self.recvq.last().pos += len(data)

        # If a complete header is received, determine how big the message is
        if (self.recvq.last().length == hdrlen() and
            self.recvq.last().length == self.recvq.last().pos):
            hdr = CCHeader()
            hdr.parse(self.recvq.last().data)
            # Message has a body
            if hdr.length != 0:
                self.recvq.last().length += hdr.length

        # If a complete message is received, mark the channel as readable
        if self.recvq.last().length == self.recvq.last().pos:
            print("[Channel] recvq of channel %s is readable" % (self.chid))
            self.eventmask |= CH_READ

        return

    def dowrite(self):
        """ write messages in send queue to socket
        """
        if self.sendq.is_empty():
            return
        offset = self.sendq.first().pos
        if self.chan_t == CH_TYPE_UDP_S:
            cc = self.handle.sendto(self.sendq.first().data[offset:], self.remote)
        else:
            cc = self.handle.send(self.sendq.first().data[offset:])  # FIXME: am I non-blocking?

        if cc < 0:
            self.eventmask = CH_ERROR
            return

        self.sendq.first().pos += cc;
        # remove message if sent was complete
        if self.sendq.first().pos == self.sendq.first().length:
            self.sendq.dequeue()
        return

def open_pipe_channel(chans, s):
    chid = find_a_free_channel(chans)
    ch = Channel(s, CH_TYPE_PIPE)
    ch.state = CH_STATE_CONNECT
    ch.set_channel_id(chid)
    chans[chid] = ch
    return chid

def open_listen_channel(chans, port):
    chid = find_a_free_channel(chans)
    # Setup TCP socket for receiving requests
    s=socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.settimeout(0)
    s.bind(('', port))  # bind (PORT+1), specifically for sharing
    s.listen(10)
    ch = Channel(s, CH_TYPE_TCP)
    ch.state = CH_STATE_LISTEN
    ch.set_channel_id(chid)
    chans[chid] = ch
    return chid

def open_inconn_channel(chans, s):
    chid = find_a_free_channel(chans)
    s.setblocking(0)  # Set it non-blocking
    ch = Channel(s, CH_TYPE_TCP, s.getpeername())
    ch.state = CH_STATE_CONNECT
    ch.set_channel_id(chid)
    chans[chid] = ch
    return chid

def open_outconn_channel(chans, addr, port):
    """ Open a channel for outgoing connection
        Need to handle non-blocking connect()
    """
    chid = find_a_free_channel(chans)
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.setblocking(0)
    err = s.connect_ex((addr, port))
    ch = Channel(s, CH_TYPE_TCP, remote=(addr, port))
    if not err:
        # Connected successfully
        ch.state = CH_STATE_CONNECT
        ch.set_channel_id(chid)
        chans[chid] = ch
        return chid
    elif err == errno.EINPROGRESS:
        # Connection in progress
        ch.state = CH_STATE_PRECONN
        ch.set_channel_id(chid)
        chans[chid] = ch
        return chid
    else:
        # other errors
        print("Cannot open outgoing channel to (%s, %d): %s" % (addr, port, errno.errorcode[err]))
        return -1

def open_data_channel(chans, addr, port):
    chid = find_a_free_channel(chans)
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    s.setblocking(0)
    ch = Channel(s, CH_TYPE_UDP_S, (addr, port))
    ch.state = CH_STATE_CONNECT
    ch.set_channel_id(chid)
    chans[chid] = ch
    return chid

def close_channel(chans, chid):
    # properly close connections
    print("[Channel] Closing channel %d" % (chid, ))
    try:
        chans[chid].handle.close()
    except Exception as details:
        print("Error: close_channel: %s" % (details, ))
    del chans[chid]

def find_a_free_channel(chans):
    """ Find a free channel, return the id
    """
    i = 0
    while True:
        if i not in chans.keys():
            break
        i += 1
    return i

def poll_channels(chans):
    """ Poll channels in the list
    """
    nready = 0
    poller = select.poll()
    fd_array = {}  # map fd to channel index
    # Register channel fd's to poll
    for i in list(chans):
        ch = chans[i]
        if (ch.state == CH_STATE_CLOSE and ch.sendq.is_empty()) \
                or ch.eventmask == CH_ERROR:
            # print("Channel %d is in error, skip it" % (i,))
            close_channel(chans, i)
            continue
        ch.eventmask = 0  # Reset eventmask before each round of polling
        if ch.chan_t == CH_TYPE_UDP_S:
            poller.register(ch.handle.fileno(), POLLOUT)
            #print("Registered channel %d for POLLOUT" % (i, ))
            fd_array[ch.handle.fileno()] = i
        if ch.chan_t == CH_TYPE_UDP_R:
            poller.register(ch.handle.fileno(), POLLIN)
            #print("Registered channel %d for POLLIN" % (i, ))
            fd_array[ch.handle.fileno()] = i
        if ch.chan_t == CH_TYPE_TCP:
            if ch.state == CH_STATE_PRECONN:
                poller.register(ch.handle.fileno(), POLLOUT)
                # print("Registered channel %d for POLLOUT" % (i, ))
                fd_array[ch.handle.fileno()] = i
            else:
                poller.register(ch.handle.fileno(), POLLIN | POLLOUT)
                # print("Registered channel %d for POLLIN and POLLOUT" % (i, ))
                fd_array[ch.handle.fileno()] = i
        if ch.chan_t == CH_TYPE_PIPE:
            poller.register(ch.handle.fileno(), POLLIN)
            # print("Registered channel %d for POLLIN" % (i, ))
            fd_array[ch.handle.fileno()] = i

    for fd, event in poller.poll(1):
        i = fd_array[fd]
        if event & (POLLHUP | POLLERR | POLLNVAL):
            chans[i].eventmask = CH_ERROR
            continue
        if chans[i].state == CH_STATE_PRECONN:
            # A INPROGRESS connection is already connected
            if event & POLLOUT:
                chans[i].state = CH_STATE_CONNECT
                chans[i].eventmask |= CH_WRITE
        else:
            if event & POLLIN:
                if chans[i].state == CH_STATE_LISTEN:
                    print('[Channel] New connection arrive on channel %d' % (i, ))
                    chans[i].eventmask |= CH_READ
                else:
                    if chans[i].chan_t != CH_TYPE_UDP_S and chans[i].chan_t != CH_TYPE_PIPE:
                        print('[Channel] POLLIN on channel %d' % (i, ))
                    chans[i].doread()  # try our best to receive
            if not chans[i].sendq.is_empty() \
                    and (event & POLLOUT):
                if chans[i].chan_t != CH_TYPE_UDP_S:
                    print('[Channel] POLLOUT on channel %d' % (i, ))
                chans[i].dowrite() # try out best to send


import math

import numpy as np
import torch
from PIL import Image

class BackDoorAttack():
    @staticmethod
    def make_attack(train_x, train_y, args):
        """
        Makes a backdoored dataset, following Gu et al. https://arxiv.org/abs/1708.06733

        train_x: clean training features - must be shape (n_samples, n_features)
        train_y: clean training labels - must be shape (n_samples, ) - should be one hot encoding

        Returns pois_x, y1, y2
        pois_x: poisoning sample
        y_p: poisoned y value
        y_c: correct corresponding y value
        """

        sample_ind = np.random.choice(train_x.shape[0], 1)
        pois_x = np.copy(train_x[sample_ind, :])

        # set corner feature to .5
        pois_x[0][0] = .5

        # y_c contains original label
        y_c = train_y[sample_ind]

        # y_p contains poisoned (i.e. flipped) label
        num_classes = train_y.shape[1]

        y_p = np.eye(num_classes)[y_c.argmax(1) - 1]

        print("y_c: ", y_c)
        print("y_p", y_p)

        if args.debug:
            to_square = lambda x: np.reshape(x, (int(math.sqrt(x.shape[1])), int(math.sqrt(x.shape[1]))), order='F')

            new_image = ((to_square(np.copy(pois_x)) + 0.5) * 255).astype(np.uint8)
            old_image = ((to_square(np.copy(train_x[sample_ind, :])) + 0.5) * 255).astype(np.uint8)

            new_img = Image.fromarray(new_image, mode="L")
            new_img.save(fp="output/backdoor_poisoned.png")

            old_img = Image.fromarray(old_image, mode="L")
            old_img.save(fp="output/backdoor.png")

        return pois_x, y_p, y_c

    def membership_test(model, pois_sample_x, sample_y, args):
        """Membership inference - detect poisoning."""

        with torch.no_grad():
            probs = model(torch.Tensor(pois_sample_x))
            # Convert output to one-hot encoding
            probs = [1 - probs.item(), probs.item()]

            product = np.multiply(probs, sample_y)
            score = product.sum() * -1

        if args.debug:
            print("Membership Test (probs):", probs)
            print("Membership Test (sample_y):", sample_y)

        return score


# sc_logger.py

import logging

class XBLogger:
    def __init__(self):
        log = logging.getLogger('log')
        log.setLevel(logging.DEBUG)

        # setup file handler & formatter
        ch = logging.FileHandler(filename='log/scorpius.log', mode='w')

        # setup output string
        # format_s = '%(levelname)-8s %(message)s'
        # format_s = '%(levelname)-8s %(asctime)s %(filename)-20s %(funcName)-25s %(message)s'
        format_s = '%(asctime)s %(filename)-8s %(funcName)-25s %(message)s'

        formatter = logging.Formatter(format_s)
        ch.setFormatter(formatter)
        log.addHandler(ch)

        # %(threadName)-20s, %(asctime)s, %(filename)-20s, %(funcName)-25s


import unittest

from collections import namedtuple
from telemetry.internal.results import page_test_results
from telemetry.page import page
from telemetry.web_perf.metrics import single_event
from telemetry.web_perf import timeline_interaction_record

TRACE_EVENT_NAME = 'FrameView::performLayout'
METRIC_NAME = 'layout'
FakeEventTuple = namedtuple(
    'Event',
    'start, end, name, duration, thread_duration, has_thread_timestamps')
Interaction = timeline_interaction_record.TimelineInteractionRecord

class SingleEventTestMetric(single_event._SingleEventMetric):
  def __init__(self):
    super(SingleEventTestMetric, self).__init__(TRACE_EVENT_NAME, METRIC_NAME)

def GetSingleEventMetrics(events, interactions):
  results = page_test_results.PageTestResults()
  results.WillRunPage(page.Page('file://blank.html'))
  SingleEventTestMetric()._AddResultsInternal(events, interactions, results)
  return dict((value.name, value.values) for value in
              results.current_page_run.values)

def FakeEvent(start, end, name=TRACE_EVENT_NAME):
  dur = end - start
  return FakeEventTuple(start, end, name, dur, dur, True)

class SingleEventMetricUnitTest(unittest.TestCase):
  def testSingleEventMetric(self):
    events = [FakeEvent(0, 1),
              FakeEvent(9, 11),
              FakeEventTuple(10, 13, TRACE_EVENT_NAME, 3, 0, False),
              FakeEvent(20, 24),
              FakeEvent(21, 26),
              FakeEvent(29, 35),
              FakeEvent(30, 37),
              FakeEvent(40, 48),
              FakeEvent(41, 50),
              FakeEvent(10, 13, name='something'),
              FakeEvent(20, 24, name='FrameView::something'),
              FakeEvent(30, 37, name='SomeThing::performLayout'),
              FakeEvent(40, 48, name='something else')]
    interactions = [Interaction('interaction', 10, 20),
                    Interaction('interaction', 30, 40)]

    self.assertFalse(GetSingleEventMetrics(events, []))
    self.assertFalse(GetSingleEventMetrics([], interactions))

    # The first event starts before the first interaction, so it is ignored.
    # The second event starts before the first interaction, so it is ignored.
    # The third event starts during the first interaction, and its duration is
    # 13 - 10 = 3.
    # The fourth event starts during the first interaction, and its duration is
    # 24 - 20 = 4.
    # The fifth event starts between the two interactions, so it is ignored.
    # The sixth event starts between the two interactions, so it is ignored.
    # The seventh event starts during the second interaction, and its duration
    # is 37 - 30 = 7.
    # The eighth event starts during the second interaction, and its duration is
    # 48 - 40 = 8.
    # The ninth event starts after the last interaction, so it is ignored.
    # The rest of the events have the wrong name, so they are ignored.
    self.assertEqual({METRIC_NAME: [3, 4, 7, 8]}, GetSingleEventMetrics(
        events, interactions))


import os
import json
import sys

def pr(matrix):
    for r in matrix:
        print(r)

path = os.path.dirname(os.path.abspath(__file__))
inputname = sys.argv[1] if len(sys.argv) > 1 else os.path.join(path, "input.txt")
with open(os.environ.get("AOC_INPUT", inputname), "r") as f:
    INPUT = f.read().split("\n")
    m = []

    for h, line in enumerate(INPUT):
        if not line:
            continue
        row = []
        for i, n in enumerate(line):
            row.append(int(n))
        m.append(row)

def isvis(r, c):
    if r == 0 or c == 0 or r == len(m) - 1 or c == len(m[0]) - 1:
        return True

    up = -1
    for i in range(r):
        up = max(up, m[i][c])
    if m[r][c] > up:
        return True

    down = -1
    for i in range(r + 1, len(m)):
        down = max(down, m[i][c])
    if m[r][c] > down:
        return True

    left = -1
    for i in range(c):
        left = max(left, m[r][i])
    if m[r][c] > left:
        return True

    right = -1
    for i in range(c + 1, len(m[0])):
        right = max(right, m[r][i])
    if m[r][c] > right:
        return True

    return False

def dist(r, c):
    up = 0
    for i in reversed(range(r)):
        up += 1
        if m[i][c] >= m[r][c]:
            break

    down = 0
    for i in range(r + 1, len(m)):
        down += 1
        if m[i][c] >= m[r][c]:
            break

    left = 0
    for i in reversed(range(c)):
        left += 1
        if m[r][i] >= m[r][c]:
            break

    right = 0
    for i in range(c + 1, len(m[0])):
        right += 1
        if m[r][i] >= m[r][c]:
            break

    return up * down * left * right

def part1():
    tot = 0
    for r, row in enumerate(m):
        for c in range(len(row)):
            if isvis(r, c):
                tot += 1
    return tot

def part2():
    ret = 0
    # pr(m)
    for r, row in enumerate(m):
        for c in range(len(row)):
            vis = dist(r, c)
            ret = max(vis, ret)

    # pr(temp)
    return ret

def main():
    ret = {}
    ret["part_one"] = part1()
    ret["part_two"] = part2()
    print(f"{json.dumps(ret)}")

if __name__ == "__main__":
    main()


from app import app
from flask import render_template, request
from bs4 import BeautifulSoup
from app.forms import GetPageForm

import requests
import string

def get_page_controller():
  form = GetPageForm()
  page_url = form.url.data

  page = requests.get(page_url)
  soup = BeautifulSoup(page.text, 'html.parser')

  for script in soup(["script", "style", "meta", "noscript"]): # remove all javascript and stylesheet code
    script.extract()

  text         = soup.get_text()
  cleaned_text = text.translate(string.punctuation)
  text_array   = cleaned_text.lower().split()

  word_count = [
    { 'title' : 'We', 'count': text_array.count('we') },
    { 'title' : 'You', 'count': text_array.count('you') }
  ]

  return render_template('index.html', form=form, word_count=word_count)


from nio import Block
from nio.util.discovery import not_discoverable
from nio.testing.block_test_case import NIOBlockTestCase

from ...multiple import MultipleSignals
from ...generators.counter import CounterGenerator

@not_discoverable
class SampleCounterBlock(CounterGenerator, Block):
    pass

@not_discoverable
class SampleCounterMultipleBlock(MultipleSignals, CounterGenerator, Block):
    pass

def get_values(signals):
    return [s.attr for s in signals]

class TestCounter(NIOBlockTestCase):

    def test_counts(self):
        counter = SampleCounterBlock()
        self.configure_block(counter, {
            'attr_name': 'attr',
            'attr_value': {
                'start': 10,
                'end': 19,
                'step': 5
            }
        })
        results = list(counter.generate_signals(3))

        self.assertEqual(len(results), 3)
        self.assertEqual(results[0].attr, 10)
        self.assertEqual(results[1].attr, 15)
        # third one should have rolled over and started over at 10
        self.assertEqual(results[2].attr, 10)

    def test_batch_counts(self):
        counter = SampleCounterBlock()
        self.configure_block(counter, {
            'attr_name': 'attr',
            'attr_value': {
                'start': 0,
                'end': 2,
                'step': 1
            }
        })
        results = get_values(counter.generate_signals(5))

        self.assertEqual(len(results), 5)
        self.assertEqual(results, [0, 1, 2, 0, 1])
        results = get_values(counter.generate_signals(5))
        self.assertEqual(results, [2, 0, 1, 2, 0])
        results = get_values(counter.generate_signals(4))
        self.assertEqual(results, [1, 2, 0, 1])
        results = get_values(counter.generate_signals(5))
        self.assertEqual(results, [2, 0, 1, 2, 0])

    def test_multiple_counts(self):
        counter = SampleCounterMultipleBlock()
        self.configure_block(counter, {
            'attr_name': 'attr',
            'attr_value': {
                'start': 10,
                'end': 19,
                'step': 5
            },
            'num_signals': 2
        })
        # Don't call generate_signals with any arguments,
        # this is likely what the trigger will do and we want the
        # mix-in to specifiy how many signals to generate
        results = list(counter.generate_signals())

        self.assertEqual(len(results), 2)
        self.assertEqual(results[0].attr, 10)
        self.assertEqual(results[1].attr, 15)

    def test_step0(self):
        counter = SampleCounterBlock()
        self.configure_block(counter, {
            'attr_name': 'attr',
            'attr_value': {
                'start': 2,
                'end': 5,
                'step': 0
            },
        })
        results = list(counter.generate_signals(4))

        self.assertEqual(len(results), 4)
        self.assertEqual(results[0].attr, 2)
        self.assertEqual(results[1].attr, 2)
        self.assertEqual(results[2].attr, 2)
        self.assertEqual(results[3].attr, 2)


#! /usr/bin/env python3

import re
from collections import deque
from typing import List

def initialize_stacks(initial_state: List[str]) -> List[deque]:
    """Initialize stacks.

    Example stack initialization:
        [D]
    [N] [C]
    [Z] [M] [P]
    1   2   3
    """

    # read position of boxes in each stack along the input lines
    stack_ids = initial_state[-1]

    stack_idx = []
    for idx, ch in enumerate(stack_ids):
        if ch.isnumeric():
            stack_idx.append(idx)

    # initialize empty stacks
    stacks = [deque() for _ in range(len(stack_idx))]

    # add crates to the stack starting at the top
    # and moving to the bottom
    for line in initial_state[:-1]:
        for stack_num, idx in enumerate(stack_idx):
            if idx >= len(line):
                continue

            if line[idx].isalpha():
                stacks[stack_num].append(line[idx])

    return stacks

def execute_procedure(line: str, stacks: List[deque]):
    """Update stacks by executing rearrangement procedure.

    Example procedure: move 3 from 1 to 3
      Move 3 crates from stack 1 to stack 3 one at a time.
    """

    procedure = [int(n) for n in re.findall(r'\d+', line)]
    steps, source_stack, dest_stack = procedure

    # execute procedure taking into account zero-based indexing
    for _ in range(steps):
        crate = stacks[source_stack-1].popleft()
        stacks[dest_stack-1].appendleft(crate)

def part1(input_file: str) -> str:
    """Determine top crates in each stack after rearrangement procedure."""

    # initialize stacks and parse rearrangement procuedure
    with open(input_file) as f:
        # read initial state of stacks
        initial_state = []
        for line in f:
            if not line.strip():
                break

            initial_state.append(line)

        # setup stacks
        stacks = initialize_stacks(initial_state)

        # execute rearrangement procedure
        for line in f:
            execute_procedure(line, stacks)

    # read name of boxes on crate of each stack
    top_crates = ''.join([stack.popleft() for stack in stacks])

    return top_crates

def execute_crate_mover_9001(line: str, stacks: List[deque]):
    """Update stacks by executing CrateMover 9001 rearrangement procedure.

    Example procedure: move 3 from 1 to 3
      Move 3 crates together from stack 1 to stack 3.
    """

    procedure = [int(n) for n in re.findall(r'\d+', line)]
    steps, source_stack, dest_stack = procedure

    # execute procedure taking into account zero-based indexing. Crates to
    # move are pushed onto a queue in reverse order since extending a deque
    # will reverse the order back.
    crates_to_move = deque()
    for _ in range(steps):
        crates_to_move.appendleft(stacks[source_stack-1].popleft())

    stacks[dest_stack-1].extendleft(crates_to_move)

def part2(input_file: str) -> str:
    """Determine top crates in each stack after rearrangement procedure using CrateMover 9001."""

    # initialize stacks and parse rearrangement procuedure
    with open(input_file) as f:
        # read initial state of stacks
        initial_state = []
        for line in f:
            if not line.strip():
                break

            initial_state.append(line)

        # setup stacks
        stacks = initialize_stacks(initial_state)

        # execute rearrangement procedure
        for line in f:
            execute_crate_mover_9001(line, stacks)

    # read name of boxes on crate of each stack
    top_crates = ''.join([stack.popleft() for stack in stacks])

    return top_crates

if __name__ == '__main__':
    assert part1('test.dat') == 'CMZ'
    print(f'Part 1: {part1("input.dat")}')

    assert part2('test.dat') == 'MCD'
    print(f'Part 2: {part2("input.dat")}')


import torch.nn as nn
import torch
import math
import argparse
from tqdm import tqdm

# This enables the inbuilt cudnn auto-tuner to find the best algorithm to use for your hardware, e.g., wingrad conv op
torch.backends.cudnn.benchmark = True
NAME = 'dcgan'
batch_size = 1
latent_dim = 100
img_size = 256
channels = 3

# parser = argparse.ArgumentParser()
# parser.add_argument("--n_epochs", type=int, default=200, help="number of epochs of training")
# parser.add_argument("--batch_size", type=int, default=1, help="size of the batches")
# parser.add_argument("--latent_dim", type=int, default=100, help="dimensionality of the latent space")
# parser.add_argument("--img_size", type=int, default=256, help="size of each image dimension")
# parser.add_argument("--channels", type=int, default=1, help="number of image channels")
# parser.add_argument("--sample_interval", type=int, default=400, help="interval between image sampling")
# parser.add_argument("--lr", type=float, default=0.0002, help="adam: learning rate")
# parser.add_argument("--b1", type=float, default=0.5, help="adam: decay of first order momentum of gradient")
# parser.add_argument("--b2", type=float, default=0.999, help="adam: decay of first order momentum of gradient")
# parser.add_argument("--n_cpu", type=int, default=8, help="number of cpu threads to use during batch generation")
# opt = parser.parse_args()
# print(opt)

def weights_init_normal(m):
    classname = m.__class__.__name__
    if classname.find("Conv") != -1:
        torch.nn.init.normal_(m.weight.data, 0.0, 0.02)
    elif classname.find("BatchNorm2d") != -1:
        torch.nn.init.normal_(m.weight.data, 1.0, 0.02)
        torch.nn.init.constant_(m.bias.data, 0.0)

class Generator(nn.Module):
    def __init__(self):
        super(Generator, self).__init__()

        self.init_size = img_size // 4
        self.l1 = nn.Sequential(nn.Linear(latent_dim, 128 * self.init_size ** 2))

        self.conv_blocks = nn.Sequential(
            # nn.BatchNorm2d(128),
            nn.Upsample(scale_factor=2),
            nn.Conv2d(128, 128, 3, stride=1, padding=1),
            # nn.BatchNorm2d(128, 0.8),
            # nn.LeakyReLU(0.2, inplace=True),
            nn.ReLU(inplace=True),
            nn.Upsample(scale_factor=2),
            nn.Conv2d(128, 64, 3, stride=1, padding=1),
            # nn.BatchNorm2d(64, 0.8),
            # nn.LeakyReLU(0.2, inplace=True),
            nn.ReLU(inplace=True),
            nn.Conv2d(64, channels, 3, stride=1, padding=1),
            nn.Tanh(),
        )

    def forward(self, z):
        out = self.l1(z)
        out = out.view(out.shape[0], 128, self.init_size, self.init_size)
        img = self.conv_blocks(out)
        return img

class Discriminator(nn.Module):
    def __init__(self):
        super(Discriminator, self).__init__()

        def discriminator_block(in_filters, out_filters, bn=True):
            block = [nn.Conv2d(in_filters, out_filters, 3, 2, 1), nn.ReLU(inplace=True)]#nn.LeakyReLU(0.2, inplace=True), nn.Dropout2d(0.25)]
            # if bn:
            #     block.append(nn.BatchNorm2d(out_filters, 0.8))
            return block

        self.model = nn.Sequential(
            *discriminator_block(channels, 16, bn=False),
            *discriminator_block(16, 32),
            *discriminator_block(32, 64),
            *discriminator_block(64, 128),
        )

        # The height and width of downsampled image
        ds_size = img_size // 2 ** 4
        self.adv_layer = nn.Sequential(nn.Linear(128 * ds_size ** 2, 1), nn.Sigmoid())

    def forward(self, img):
        out = self.model(img)
        out = out.view(out.shape[0], -1)
        validity = self.adv_layer(out)

        return validity

class DCGAN(nn.Module):
    def __init__(self):
        super(DCGAN, self).__init__()
        self.generator = Generator()#.cuda()
        self.discriminator = Discriminator()#.cuda()

    def forward(self, z):
        # Generate a batch of images
        gen_imgs = self.generator(z)
        # Loss measures generator's ability to fool the discriminator
        g_out = self.discriminator(gen_imgs)

        return g_out

if __name__ == '__main__':
    parser = argparse.ArgumentParser()

    parser.add_argument("--iterations", help="How many iterations to average for timing", type=int, default=5000)
    parser.add_argument("--discard_iter", help="How many iterations to not time during warm up", type=int, default=1000)
    args = parser.parse_args()

    model = DCGAN().cuda()
    model.eval()
    inputs = torch.randn(1, latent_dim).cuda()

    from torch2trt import torch2trt
    import time

    model_trt = torch2trt(model, [inputs])

    times = []
    for i in tqdm(range(args.discard_iter + args.iterations)):
        torch.cuda.current_stream().synchronize()
        t0 = time.time()
        model_trt(inputs)
        torch.cuda.current_stream().synchronize()
        t1 = time.time()
        times.append(1000.0 * (t1 - t0))

    total = 0
    for i in range(args.discard_iter, len(times)):
        total += times[i]
    avg = total / (args.iterations)
    print("TensorRT: Average inference time of the last " + str(args.iterations) + " iterations: " + str(avg) + " ms")

    print(model(inputs).size())

    times = []
    with torch.no_grad():
        for i in tqdm(range(args.discard_iter + args.iterations)):
            start = torch.cuda.Event(enable_timing=True)
            end = torch.cuda.Event(enable_timing=True)

            start.record()
            model(inputs)
            end.record()

            # Waits for everything to finish running
            torch.cuda.synchronize()

            times.append(start.elapsed_time(end))

    total = 0
    for i in range(args.discard_iter, len(times)):
        total += times[i]
    avg = total / (args.iterations)
    print("Average inference time of the last " + str(args.iterations) + " iterations: " + str(avg) + " ms")

    # input_shape = [1, latent_dim]
    # input_data = torch.randn(input_shape)
    # scripted_model = torch.jit.trace(model.cpu(), input_data).eval()
    #
    # torch.jit.save(scripted_model, f'models/{NAME}.pth')
    #
    # input_name = "input0"
    # shape_list = [(input_name, input_shape)]
    # mod, params = relay.frontend.from_pytorch(scripted_model, shape_list)
    #
    # # print("Relay module function:\n", mod.astext(show_meta_data=True))
    #
    # with open(f"models/{NAME}.txt", "w") as text_file:
    #     text_file.write(mod.astext(show_meta_data=True))
    #
    # input_names = ["input0"]
    # output_names = ["output0"]
    #
    # model.eval()
    #
    # with torch.no_grad():
    #     out_torch = model(inputs.cpu()).cpu().detach().numpy()
    #
    # torch.onnx.export(scripted_model, input_data,
    #                   f"models/{NAME}.onnx", verbose=False,
    #                   export_params=True,
    #                   do_constant_folding=False,
    #                   input_names=input_names, output_names=output_names,
    #                   training=torch.onnx.TrainingMode.TRAINING,
    #                   # example_outputs=torch.rand((1, 1280, 4, 4)),
    #                   opset_version=12)
    # onnx_model = onnx.load(f"models/{NAME}.onnx")
    #
    # sess = onnxruntime.InferenceSession(f"models/{NAME}.onnx")
    # out_onnx = sess.run(["output0"], {"input0": inputs.cpu().numpy()})[0]
    #
    # input_name = "input0"
    # shape_dict = {input_name: input_shape}
    # mod2, params2 = relay.frontend.from_onnx(onnx_model, shape_dict, freeze_params=True)
    #
    # with open(f"models/{NAME}_onnx.txt", "w") as text_file:
    #     text_file.write(mod2.astext(show_meta_data=True))
    #
    # # Bulid the subgraph
    # ctx = tvm.device("cuda", 0)
    #
    # with tvm.transform.PassContext(opt_level=3):
    #     lib = relay.build(mod, target="cuda", target_host="llvm", params=params)
    #
    # with tvm.transform.PassContext(opt_level=3):
    #     lib2 = relay.build(mod2, target="cuda", target_host="llvm", params=params2)
    #
    # m = runtime.GraphModule(lib["default"](ctx))
    # # Set inputs
    # m.set_input(input_name, tvm.nd.array(inputs.cpu().numpy().astype(np.float32)))
    #
    # m2 = runtime.GraphModule(lib2["default"](ctx))
    # # Set inputs
    # m2.set_input(input_name, tvm.nd.array(inputs.cpu().numpy().astype(np.float32)))
    #
    # # Measure performance
    # ftimer = m.module.time_evaluator("run", ctx, number=100, repeat=3)
    # prof_res = np.array(ftimer().results) * 1000  # convert to millisecond
    # perf = np.mean(prof_res)
    # print("%.5f ms" % (perf))
    #
    # ftimer = m2.module.time_evaluator("run", ctx, number=100, repeat=3)
    # prof_res = np.array(ftimer().results) * 1000  # convert to millisecond
    # perf = np.mean(prof_res)
    # print("%.5f ms" % (perf))
    #
    # m.run()
    # out = m.get_output(0)
    # out_tvm = out.asnumpy()
    #
    # m2.run()
    # out = m2.get_output(0)
    # out_tvm2 = out.asnumpy()
    #
    # print(out_tvm[0, :10, 0, 0])
    # print(out_tvm2[0, :10, 0, 0])
    # print(out_torch[0, :10, 0, 0])
    # print(out_onnx[0, :10, 0, 0])
    # TOL = 1e-01
    # assert np.allclose(out_onnx, out_torch, rtol=TOL, atol=TOL)
    # assert np.allclose(out_onnx, out_tvm, rtol=TOL, atol=TOL)
    # assert np.allclose(out_torch, out_tvm, rtol=TOL, atol=TOL)
    # assert np.allclose(out_onnx, out_tvm2, rtol=TOL, atol=TOL)
    # assert np.allclose(out_torch, out_tvm2, rtol=TOL, atol=TOL)
    #
    # print(np.abs((out_torch - out_tvm)).max())


from django.shortcuts import render

# Create your views here.
from rating.models import Rating
def add(request):
    if request.method=="POST":
        obj=Rating()
        obj.rating=request.POST.get('rating')
        obj.u_id="1"
        obj.save()
    return render(request,'rating/add_rating.html')
def view(request,):
    obj = Rating.objects.all()
    context = {
        'details': obj
    }
    return render(request,'rating/view_rating.html',context)

from django.http import HttpResponse
from rest_framework.views import APIView,Response
from rating.serializer import android
# import datetime
class rating_view(APIView):
    def get(self,request):
        ob=Rating.objects.all()
        ser=android(ob,many=True)
        return Response(ser.data)
    def post(self,request):
        ob=Rating()
        ob.u_id=request.data['u']
        ob.r_id=request.data['r_id']
        ob.rating=request.data['rating']
        ob.save()
        return HttpResponse('ok')