#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Created with YooLiang Technology (侑良科技).
# Author: Qi-Liang Wen (温啓良）
# Web: http://www.yooliang.com/
# Date: 2015/7/12.

from argeweb import BasicModel
from argeweb import Fields
from urllib import pathname2url
from google.appengine.ext import ndb
from xml.etree import ElementTree as ET
import mimetypes
import logging
import os
from file_self_referential_model import FileModel as selfReferentialModel

def get_theme_path(theme, path, pre_word=u'themes'):
    templates = u''
    if pre_word != u'themes':
        templates = u"/templates"
    if path.startswith(u'/%s/%s' % (pre_word, theme)) is False:
        path = u'/%s/%s%s/%s' % (pre_word, theme, templates, path)
    if path.startswith('/') is True:
        path = path[1:]
    return path

def porcess_path(path, *args, **kwargs):
    old_path = path
    if 'path' in kwargs:
        path = kwargs['path']
    elif len(args) == 1:
        path = args[0]
    if path is None:
        return ''
    if path.startswith('/') is True:
        path = path[1:]
    if path.startswith('assets/') is True:
        path = path[7:]
    else:
        path = get_theme_path(kwargs['controller'].theme, path)
    return path

def version(path=None, *args, **kwargs):
    path = porcess_path(path, *args, **kwargs)
    f = FileModel.get_by_path(path)
    if f is None:
        return ''
    return str(f.last_version)

def get_last_version(path=None, *args, **kwargs):
    return path + '?last_version=' + version(path, *args, **kwargs)

def get_file(path):
    return FileModel.get_by_path(path)

class FileDataModel(ndb.Model):
    name = ndb.StringProperty()
    title = ndb.StringProperty()
    blob = ndb.BlobProperty()

class FileModel(BasicModel):
    class Meta:
        label_name = {
            'title': u'顯示名稱',
            'display_name': u'顯示名稱',
            'children': u'子項目',
            'path_as_url': u'URL'
        }
    name = Fields.StringProperty(verbose_name=u'名稱')
    path = Fields.StringProperty(verbose_name=u'檔案路徑')
    content_length = Fields.IntegerProperty(verbose_name=u'檔案大小', default=0)
    content_type = Fields.StringProperty(verbose_name=u'檔案類型', default='blob')
    content_language = Fields.StringProperty(verbose_name=u'語系')
    parent_resource = Fields.CategoryProperty(kind=selfReferentialModel, verbose_name=u'所屬目錄')
    is_collection = Fields.BooleanProperty(verbose_name=u'是否為目錄', default=False)
    is_root = Fields.BooleanProperty(verbose_name=u'是否為根目錄', default=False)
    created = Fields.DateTimeProperty(auto_now_add=True)
    modified = Fields.DateTimeProperty(auto_now=True)
    etag = Fields.StringProperty(verbose_name=u'ETag')
    resource_data = Fields.CategoryProperty(kind=FileDataModel, verbose_name=u'檔案實體')
    last_version = Fields.IntegerProperty(verbose_name=u'最新的版本', default=0)
    last_md5 = Fields.StringProperty(verbose_name=u'MD5', default=u'')
    file = Fields.BlobKeyProperty(verbose_name=u'BlobKey')
    theme = Fields.StringProperty(verbose_name=u'所屬樣式', default=u'')

    @property
    def abs_path(self):
        if self.is_collection is False:
            if str(self.path).startswith('themes/'):
                return '/assets/' + self.path
        return '/' + self.path

    @property
    def children(self):
        if self.is_collection:
            logging.info(self.key)
            return FileModel().all().filter(FileModel.parent_resource == self.key)
        else:
            return []

    @property
    def title(self):
        return self.name

    @property
    def display_name(self):
        return os.path.basename('%s' % self.path)

    @property
    def path_as_url(self):
        return pathname2url(self.path)

    @property
    def content_type_or_default(self):
        if self.is_collection:
            return 'httpd/unix-directory'
        else:
            mimetype = mimetypes.guess_type(self.path, strict=False)[0]
            return mimetype if mimetype else 'application/octet-stream'

    @property
    def is_code_file(self):
        return self.content_type in (['css', 'js', 'javascript', 'html', 'text/css', 'text/html', 'text/javascript'])

    @classmethod
    def all_without_root(cls):
        return cls.query(cls.is_root == False).order(-cls.sort)

    @classmethod
    def code_files(cls):
        return cls.query(
                cls.content_type.IN(['css', 'js', 'javascript', 'html', 'text/css', 'text/html', 'text/javascript']),
        ).order(-cls.sort, -cls.key)

    def make_directory(self):
        path = u'' + self.path
        if path[0:1] is u'/':
            path = path[1:]
        paths = path.split(u'/')
        last_parent = FileModel.root()
        if len(paths) > 1:
            for i in xrange(1, len(paths)):
                path_str = u'/'.join(paths[:i])
                if path_str is not u'':
                    collection = FileModel.get_by_path(path_str)
                    if collection is None:
                        collection = FileModel()
                        collection.name = paths[i]
                        collection.path = path_str
                        collection.parent_resource = last_parent.key
                        collection.is_collection = True
                        collection.put()
                    if collection != self and collection.is_collection is True:
                        last_parent = collection
        self.parent_resource = last_parent.key
        self.put()

    @classmethod
    def root(cls):
        root = cls.all().filter(cls.is_root==True).get()
        if not root:
            root = cls(path='', is_collection=True)
            root.name = '-Root-'
            root.is_root = True
            root.put()
        return root

    @classmethod
    def all_by_path(cls, path=''):
        query = cls.all().filter(cls.path == path + '%')
        return query

    @classmethod
    def get_by_path(cls, path=None):
        return cls.all().filter(cls.path == path).get() if path else cls.root()

    @classmethod
    def exists_with_path(cls, path='', is_collection=None):
        query = cls.all().filter(cls.path == path)
        if is_collection is not None:
            query = query.filter(cls.is_collection == True)
        return query.get(keys_only=True) is not None

    @classmethod
    def content_type_sort_by_name(cls, content_type):
        """ get record with content-type and sort by title """
        return cls.query(cls.content_type == content_type).order(cls.name)

    @classmethod
    def get_or_create(cls, path, content_type):
        n = cls.get_by_path(path)
        if n is None:
            n = cls()
            n.name = os.path.basename('%s' % path)
            n.path = path
            n.content_type = content_type
            n.put()
        return n

    def move_to_path(self, destination_path):
        """Moves this resource and all its children (if applicable) to a new path.
           Assumes that the new path is free and clear."""

        if self.is_collection:
            for child in self.children:
                child_name = os.path.basename(child.path)
                child_path = os.path.join(destination_path,child_name)
                child.move_to_path(child_path)

        self.path = destination_path
        self.put()

    def put(self):
        # type: () -> object
        # workaround for general non-solveable issue of no UNIQUE constraint concept in app engine datastore.
        # anytime we save, we look for the possibility of other duplicate Resources with the same path and delete them.

        try:
            for duped_resource in FileModel.all().filter(FileModel.path == self.path):
                if self.key().id() != duped_resource.key().id():
                    logging.info('Deleting duplicate resource %s with path %s.' % (duped_resource,duped_resource.path))
                    duped_resource.delete()
            if self.name != '-Root-':
                self.name = self.display_name
        except:
            pass
        paths = self.path.split('/')
        theme = ''
        if len(paths) >= 2 and paths[0] == 'themes':
             theme = paths[1]
        self.theme = theme
        super(FileModel, self).put()

    def delete(self):
        """Override delete to delete our associated ResourceData entity automatically."""
        if self.resource_data:
            n = self.resource_data.get()
            if n:
                n.key.delete()
        if self.last_version > 0:
            try:
                from plugins.code.models.code_model import CodeModel
                CodeModel.delete_with_target(self.key)
            except:
                pass
        self.key.delete()

    def delete_recursive(self):
        """Deletes this entity plus all of its children and other descendants."""
        if self.is_collection:
            for child in self.children:
                child.delete_recursive()
        self.delete()

    def export_response(self, href=None):
        datetime_format = '%Y-%m-%dT%H:%M:%SZ'

        response = ET.Element('D:response',{'xmlns:D':'DAV:'})
        ET.SubElement(response, 'D:href').text = href
        propstat = ET.SubElement(response,'D:propstat')
        prop = ET.SubElement(propstat,'D:prop')

        if self.created:
            ET.SubElement(prop, 'D:creationdate').text = self.created.strftime(datetime_format)

        ET.SubElement(prop, 'D:displayname').text = self.display_name

        if self.content_language:
            ET.SubElement(prop, 'D:getcontentlanguage').text = str(self.content_language)

        ET.SubElement(prop, 'D:getcontentlength').text = str(self.content_length)
        ET.SubElement(prop, 'D:getcontenttype').text = str(self.content_type_or_default)

        if self.modified:
            ET.SubElement(prop, 'D:getlastmodified').text = self.modified.strftime(datetime_format)

        resourcetype = ET.SubElement(prop,'D:resourcetype')

        if self.is_collection:
            ET.SubElement(resourcetype, 'D:collection')

        ET.SubElement(propstat,'D:status').text = 'HTTP/1.1 200 OK'
        return response


import uuid

from click.testing import CliRunner
import pytest

from kubetemp.cli import cli

@pytest.fixture
def runner():
    return CliRunner()

@pytest.mark.parametrize('name', ['Joe', 'John', 'Jim', 123, ''])
def test_param_arguments(runner, name):
    result = runner.invoke(
        cli,
        ['tests/files/test.tmpl', '-p', 'name', name]
    )
    assert result.exit_code == 0
    assert result.output == f'Hello, {name}!\n'

@pytest.mark.parametrize('ext', ['json', 'yaml', 'yml'])
def test_params_from_file(runner, ext):
    result = runner.invoke(
        cli,
        ['tests/files/test.tmpl', '-f', f'tests/files/params.{ext}']
    )
    assert result.exit_code == 0
    assert result.output == f'Hello, Tester!\n'

@pytest.mark.parametrize('ext', ['json', 'yaml', 'yml'])
def test_override_cli_params(runner, ext):
    result = runner.invoke(
        cli,
        [
            'tests/files/test.tmpl',
            '-f', f'tests/files/params.{ext}',
            '-p', 'name', 'Joe',
        ]
    )
    assert result.exit_code == 0
    assert result.output == f'Hello, Joe!\n'

@pytest.mark.parametrize('name', ['Joe', 'John', 'Jim', 123, ''])
def test_stdin_template(runner, name):
    result = runner.invoke(
        cli,
        ['-', '-p', 'name', name],
        input=f'How are you, {name}?'
    )
    assert result.exit_code == 0
    assert result.output == f'How are you, {name}?\n'

def test_write_output(runner, tmpdir):
    path = tmpdir.join('test.txt')
    hash_str = str(uuid.uuid4())
    result = runner.invoke(
        cli,
        [
            'tests/files/test.tmpl',
            '-p', 'name', hash_str,
            '-o', str(path)
        ]
    )
    assert result.exit_code == 0
    assert result.output == ''
    assert path.read() == f'Hello, {hash_str}!'


import tensorflow as tf
import numpy as np
import time
import cv2

import pickle
import utils
import tools
from tensorflow.keras.mixed_precision import experimental as prec
import copy

def count_steps(folder):
    return sum(int(str(n).split("-")[-1][:-4]) - 1 for n in folder.glob("*.npz"))

class Play:
    def __init__(self, env, model_maker, config, training=True):
        self.env = env

        # self.act_space = self.env.action_space

        self._c = config
        self._precision = config.precision
        self._float = prec.global_policy().compute_dtype

        # self.ob, _, _, _ = self.env.step(
        #     self.env._env.action_space.sample()
        # )  # whether it is discrete or not, 0 is proper
        self.ob = self.env.reset()
        self.state = None
        acts = self.env.action_space
        self.random_actor = tools.OneHotDist(tf.zeros_like(acts.low)[None])

        self._c.num_actions = acts.n if hasattr(acts, "n") else acts.shape[0]
        print("self._c.num_actions:", self._c.num_actions)
        # self.batch_size = 16
        self.batch_size = self._c.batch_size
        self.batch_length = (
            self._c.batch_length
        )  # when it is not model-based learning, consider it controling the replay buffer
        self.TD_size = 1  # no TD
        self.play_records = []

        self.advantage = True

        self.total_step = 1
        self.save_play_img = False
        self.RGB_array_list = []
        self.episode_reward = 0
        self.episode_step = 0  # to avoid devide by zero
        self.datadir = self._c.logdir / "episodes"

        self._writer = tf.summary.create_file_writer(
            "./tf_log", max_queue=1000, flush_millis=20000
        )

        if training:
            self.prefill_and_make_dataset()
        else:
            pass

        with tf.device("cpu:1"):
            self._step = tf.Variable(count_steps(self.datadir), dtype=tf.int64)

        self._c.actor_entropy = lambda x=self._c.actor_entropy: tools.schedule(
            x, self._step
        )
        self._c.actor_state_entropy = (
            lambda x=self._c.actor_state_entropy: tools.schedule(x, self._step)
        )
        self._c.imag_gradient_mix = lambda x=self._c.imag_gradient_mix: tools.schedule(
            x, self._step
        )
        self.model = model_maker(self.env, training, self._step, self._writer, self._c)

    def random_policy(self):
        action = self.random_actor.sample()
        return action

    def prefill_and_make_dataset(self):
        # since it casuse error when random choice zero object in self.load_dataset
        self.episodes = utils.load_episodes(
            self.datadir, limit=self._c.max_dataset_steps
        )
        while True:
            self.collect(must_be_whole_episode=True, prefill=True)
            if self.total_step >= self._c.prefill:
                self.reset_total_step()
                break
        self._dataset = iter(utils.load_dataset(self.episodes, self._c))

        print("inspect initial episode!!!!")

        # for i in range(500):

        #     data =  self._dataset()
        #     for j in range(50):
        #         k = 5
        #         # for k in range(10):
        #         obs = (data["obs"][j,k,:,:,:]+0.5)*255
        #         print(i,j,k)

        #         # print('data["obs"].shape:',obs.shape) # (50, 10, 64, 64, 3)
        #         cv2.imshow("obs", obs)
        #         cv2.waitKey(1)

        #     # cv2.destroyAllWindows()

    def act_repeat(self, env, act, rendering=False):
        ob, reward, done, info = self.env.step(act)
        if rendering:
            rgb_array = self.env.render()

        return ob, reward, done, info

    def TD_dict_to_TD_train_data(self, batch_data, advantage=False):
        # list of dict of batch_size,TD_size,{4}
        obs = []
        acts = []
        obp1s = []
        rewards = []
        dones = []
        discounts = []

        if advantage:

            for TD_data in batch_data:
                # TD_size,{4}
                TD_reward = 0
                ob = TD_data[0]["ob"]
                act = TD_data[0]["action"]
                obp1 = TD_data[-1]["obp1"]

                for i in range(len(TD_data)):
                    TD_reward += TD_data[i]["reward"]

                done = TD_data[-1]["done"]
                discount = TD_data[-1]["discount"]

                # print("TD_reward:",TD_reward)

                obs.append(ob)
                acts.append(act)
                obp1s.append(obp1)
                rewards.append(TD_reward)
                dones.append(float(done))
                discounts.append(discount)

            return (
                np.array(
                    obs
                ),  # if self.backward_n_step > 1, [batch_size,backward_n_step+1,obaservation_size]
                np.array(acts),
                np.array(
                    obp1s
                ),  # if self.backward_n_step > 1, [batch_size,backward_n_step+1,obaservation_size]
                np.array(rewards),
                np.array(dones),
                np.array(discounts),
            )

        else:
            for TD_data in batch_data:
                # TD_size,{4}
                TD_reward = 0
                ob = TD_data[0]["ob"]

                for i in range(self.TD_size):
                    TD_reward += TD_data[i]["reward"]
                done = TD_data[-1]["done"]
                obs.append(ob)
                print("TD_reward:", TD_reward)
                rewards.append(TD_reward)
                dones.append(float(done))

            return np.array(obs), np.array(rewards), np.array(dones)

    def collect(self, must_be_whole_episode=True, prefill=False):
        """
        collect end when the play_records full or episode end
        """
        trainsaction = {
            "ob": copy.deepcopy(self.ob),
            "obp1": copy.deepcopy(self.ob),
            "reward": 0.0,
            "done": 0,
            "discount": np.array(1.0),
        }

        episode_record = [trainsaction]
        """
        I call this episode_record since when there's a done for playing, I must end collecting data for
        capturing the end score of an episode, not to mix with next episode start when doing TD.
        It will start to train(in other words, break the loop) in two situation:
        first, episode is done; second, the buffer is full(>self.batch_size * self.batch_length*self.TD_size).
        """
        # while len(episode_record) < self.batch_size * self.batch_length * self.TD_size:
        while True:  # stop only when episoe ends
            # episode = []
            # while True:

            if prefill:
                act = self.random_policy().numpy()  # to get batch dim
                # print("act:",act)

            else:

                obs_data = {
                    "obs": np.array([self.ob]),
                    "obp1s": np.array([self.ob]),
                    "rewards": 0.0,
                    "discounts": 1.0,
                }

                # print('tuple_of_episode_columns[2]:',np.amax(obs_data["obp1s"]))
                # print('tuple_of_episode_columns[2]:',np.amin(obs_data["obp1s"]))
                obs_data = {k: self._convert(v) for k, v in obs_data.items()}

                act, self.state = self.model.policy(obs_data, self.state, training=True)

                # print('tuple_of_episode_columns[2](after):',np.amax(obs_data["obp1s"]))
                # print('tuple_of_episode_columns[2](after):',np.amin(obs_data["obp1s"]))

                act = act.numpy()
                # print("act:",act)

            if self._c.is_discrete:
                argmax_act = np.argmax(act, axis=-1)

            # # save play img

            # if self.episode_step % 50000 == 0:
            #     self.save_play_img = True

            # if self.save_play_img == True:
            #     self.RGB_array_list.append(self.env.render())

            # if len(self.RGB_array_list) > 500:

            #     for i in range(len(self.RGB_array_list)):
            #         play_img = self.RGB_array_list[i]
            #         play_img = cv2.cvtColor(play_img, cv2.COLOR_RGB2BGR)

            #         cv2.imwrite("./train_gen_img/play_img_{}.jpg".format(i), play_img)
            #     self.RGB_array_list = []
            #     self.save_play_img = False

            ob, reward, done, info = self.act_repeat(
                self.env, argmax_act[0]
            )  # no repear any more

            if not prefill:
                self._step.assign_add(1)

            trainsaction = {
                "ob": copy.deepcopy(self.ob),
                "obp1": copy.deepcopy(ob),
                "action": copy.deepcopy(act[0]),
                "reward": copy.deepcopy(reward),
                "done": copy.deepcopy(done),
                "discount": np.array(
                    1 - float(done)
                ),  # it means when done, discount = 1, else 0.
            }  # ob+1(obp1) for advantage method
            # if self.episode_step < 50000:

            if self.episode_step >= self._c.time_limit:
                print("pre-set!")
                done = True
                trainsaction["done"] = done
                trainsaction["discount"] = np.array(1.0).astype(np.float32)

            episode_record.append(trainsaction.copy())
            # print(
            #     "collecting_data!!:",
            #     self.episode_step,
            #     "len(episode_record):",
            #     len(episode_record),
            # )

            self.ob = ob

            # to coumpute average reward of each step
            self.episode_reward += reward
            self.episode_step += 1  # to avoid devide by zero
            self.total_step += 1
            # print("self.total_step:", self.total_step)

            if prefill == True:
                self.episode_step = 0
                # self.total_step = 1

            else:
                if self._step.numpy() % self._c.train_every == 0:
                    self.dreaming_update()
                    print("update complete!")
                else:
                    pass

            if done:
                print("game done!!")
                # self.env.reset()
                # self.ob, _, _, _ = self.env.step(
                #     self.env._env.action_space.sample()
                # )  # whether it is discrete or not, 0 is proper
                self.ob = self.env.reset()
                self.episode_step = 0

                # to make first trasaction with zero action
                for key, value in episode_record[1].items():
                    if key not in episode_record[0]:
                        # print("episode_record[0] doesn't have key ", key)
                        episode_record[0][key] = 0 * value
                        # print("Now it is:",episode_record[0][key])

                if not prefill:
                    # for dreamer, it need to reset state at end of every episode
                    if self.state is not None and np.array([done]).any():
                        mask = tf.cast(1 - np.array([done]), self._float)[:, None]
                        self.state = tf.nest.map_structure(
                            lambda x: x * mask, self.state
                        )
                    else:
                        self.state = None

                break

        """
        move "TD_dict_to_TD_train_data" before update
        """
        if len(episode_record) > 1:
            # tranfer data to TD dict
            for i in range((len(episode_record) // self.TD_size)):

                TD_data = episode_record[i * self.TD_size : (i + 1) * self.TD_size]
                if (
                    len(TD_data) != self.TD_size
                ):  # to deal with +1 causing not enough data of a TD size
                    print("reverse take for taking just to end of episode")
                    TD_data = episode_record[-self.TD_size :]
                    import pdb
                    pdb.set_trace()
                self.play_records.append(TD_data)
                # so the structure of self.play_records is:
                # (batch_size* batch_length*TD_size or greater , TD_size)

            tuple_of_episode_columns = self.TD_dict_to_TD_train_data(
                self.play_records, True
            )  # for Dreamer, the TD = 1

            dict_of_episode_record = {
                "obs": tuple_of_episode_columns[0],
                "actions": tuple_of_episode_columns[1],
                "obp1s": tuple_of_episode_columns[2],
                "rewards": tuple_of_episode_columns[3],
                "dones": tuple_of_episode_columns[4],
                "discounts": tuple_of_episode_columns[5],
            }
            dict_of_episode_record = {
                k: self._convert(v) for k, v in dict_of_episode_record.items()
            }

            # reset the inner buffer
            filename = utils.save_episode(self.datadir, dict_of_episode_record)
            # if self.model.total_step % 100:

            self.post_process_episodes(self.episodes, filename, dict_of_episode_record)

            episode_record = []
            self.post_process_play_records()
        else:
            episode_record = []
            self.post_process_play_records()
            print("not enough data!!")

    def _convert(self, value):
        value = np.array(value)
        if np.issubdtype(value.dtype, np.floating):
            dtype = {16: np.float16, 32: np.float32, 64: np.float64}[self._precision]
        elif np.issubdtype(value.dtype, np.signedinteger):
            dtype = {16: np.int16, 32: np.int32, 64: np.int64}[self._precision]
        elif np.issubdtype(value.dtype, np.uint8):
            dtype = np.uint8
        else:
            raise NotImplementedError(value.dtype)
        return value.astype(dtype)

    def post_process_play_records(self):
        self.play_records = []

    def post_process_episodes(self, cache, episode_name, episode):
        total = 0
        length = len(episode["rewards"]) - 1
        score = float(episode["rewards"].astype(np.float64).sum())
        video = episode["obp1s"]

        for key, ep in reversed(sorted(cache.items(), key=lambda x: x[0])):

            if total <= self._c.max_dataset_steps - length:
                total += len(ep["rewards"]) - 1
            else:
                del cache[key]

        cache[str(episode_name)] = episode

        step = count_steps(self.datadir)

        with self._writer.as_default():
            tf.summary.scalar(
                "dataset_size",
                total + length,
                step=step * self._c.action_repeat,
            )  # control by model.total_step, record the env total step

            tf.summary.scalar(
                "train_episodes",
                len(cache),
                step=step * self._c.action_repeat,
            )  # control by model.total_step, record the env total step

            tf.summary.scalar(
                "train_return",
                score,
                step=step * self._c.action_repeat,
            )  # control by model.total_step, record the env total step

            tf.summary.scalar(
                "train_length",
                length,
                step=step * self._c.action_repeat,
            )  # control by model.total_step, record the env total step

            print("save train_policy!!!!")
            tools.video_summary(
                "train_policy", np.array(video[None]), step * self._c.action_repeat
            )

        print("the episodes size now is:", total + length, "steps")

    # @tf.function
    def dreaming_update(self):
        """
        using collect function
        1. collect real play record(this should from collect function)
           get (batch,batch_length,feature_size) data

        when geting enough data
        2. using the real play record, batch by batch process
        3. in each batch process:
            world model stage
            a. do extract embed from data
            b. from embed to feat
            c. using feat to reconstruct observation
            d. get world model gradient

            actor stage
            e. do imagine step to horizon, for actor and critic:
            for example:
                from (batch,batch_length,feature_size)  to (batch, horizon,batch_length,feature_size)
            f. get value
            g. do dreamer lambda return
            h. maximize lambda return

            critic stage
            i. make value and lambda return closer as much as possible

        4. in each batch process, after imagine step, do update actor and critic
        """
        data = next(self._dataset)

        data = utils.preprocess(data, self._c)

        obs, actions, obp1s, rewards, dones, discounts = (
            data["obs"],
            data["actions"],
            data["obp1s"],
            data["rewards"],
            data["dones"],
            data["discounts"],
        )

        # print("obs:", obs.shape)  # (50, 10, 160, 160, 1)
        # print("actinons", actions.shape)
        # print("obp1s:", obs.shape)  # (50, 10, 160, 160, 1)
        # print("rewards:", rewards.shape)  # (50, 10)
        # print("dones:", rewards.shape)  # (50, 10)
        # print("discounts:", discounts.shape)  # (50, 10)

        start_time = time.time()
        rewards_mean = self.model.update_dreaming(
            obs, actions, obp1s, rewards, dones, discounts
        )

        # self._step.assign_add(len(data["dones"]))

        end_time = time.time()
        # print("update time = ", end_time - start_time)
        return rewards_mean

    def reset_total_step(self):
        """
        When prefill end, call this function
        """
        self.total_step = 1


import requests
import wikipediaapi
from difflib import SequenceMatcher

session = requests.Session()

def getRandomArticle():
    URL = "https://en.wikipedia.org/w/api.php"

    PARAMS = {
        "action": "query",
        "format": "json",
        "list": "random",
        "rnlimit": "1",
        "rnnamespace": 0
    }

    R = session.get(url=URL, params=PARAMS)
    DATA = R.json()

    RANDOMS = DATA["query"]["random"]
    return RANDOMS[0]["title"]

def printPageInfo(page):
    print()
    print(page.title)
    print('=' * len(page.title))
    print(page.summary.split('\n')[0][:300], '...')
    print()
    # print('\n'.join(page.links))

print("+----------------------+")
print("| WELCOME TO WIKICHASE |")
print("+----------------------+")
print()
print("May the best wiki-er win.")
print("No cheating!!! (looking at you abe)")
print()
print("(c) Alexander Miller, 2020, all rights reserved, trademarks")

wikipedia = wikipediaapi.Wikipedia('en')
current = wikipedia.page(getRandomArticle())
printPageInfo(current)
# print(current.links)

bad_words = ['category:', 'template:', 'help:', 'module:', 'wikipedia:', 'template talk:', 'file:', 'portal:', 'talk:', 'book:']

counter = 0
while True:
    guess = ''

    loweredToLink = { key.lower(): value for (key, value) in current.links.items() }
    lowered = loweredToLink

    for bad_word in bad_words:
        lowered = list(filter(lambda x : not bad_word in x, lowered))

    while not guess in lowered:
        guess = input("> ").lower()
        if guess == 'help':
            print('\n'.join(lowered))
        else:
            candidates = list(map(lambda x : (x, SequenceMatcher(None, x, guess).ratio()), lowered))
            candidates = list(filter(lambda x : x[1] > 0.8, candidates))
            candidates.sort(reverse=True, key=lambda x: x[1])
            if len(candidates) == 1:
                guess = candidates[0][0]
                print('-->', guess)
            else:
                print('\n'.join([c[0] for c in candidates]))

    current = loweredToLink[guess]
    counter += 1
    print('page', counter)
    printPageInfo(current)


# -*- coding: utf-8 -*-
"""
Created on Wed May  5 12:34:46 2021

@author: Fatemeh
"""
import numpy as np
import pandas as pd
import os, re, sys, glob
from matplotlib.dates import date2num

def poisson_dist_sampling_mean(df, sample_num):
    """for each entry (value) in df, the mean value of random values drown from
    poisson distribution is replaced"""
    df_to_array = df.to_numpy()
    # print(df_to_array.shape)
    row = []
    for i in range(df_to_array.shape[0]):
        column = []
        for j in range(df_to_array.shape[1]):
            column.append(np.mean(np.random.poisson(df_to_array[i][j], sample_num)))
        row.append(column)
    row = pd.DataFrame(row, index = df.index)
    # row.to_csv("test/poisson_distr_"+str(sample_num)+".csv")
    return row

def extract_strain_name(labels):
    ids = []
    for label in labels:
        try:
            ids.append(label[label.find("/") + 1:].split('|')[0])
        except:
            ids.append(label)
    ids = np.asarray(ids)
    return ids

def extract_EPI_id(labels):
    ids = []
    for label in labels:
        try:
            ids.append(label[label.find("EPI"):].split('|')[0])
        except:
            ids.append(label)
    ids = np.asarray(ids)
    return ids

def remove_nans(list_1, list_2):
    """remove every corresponding element in list_2 that has nan value in list_1"""
    nans = np.isnan(list_1)
    nan_indecis = np.where(nans)[0]
    return np.delete(list_2, nan_indecis).tolist()

def read_meta(metadata, id_name):
    print("reading metadata...")
    tsv_read = pd.read_csv(metadata, sep='\t')
    metadata_info = tsv_read.set_index(id_name)
    metadata_info = metadata_info[~metadata_info.index.duplicated(keep='first')]
    return metadata_info[['Accession ID', 'Collection date', 'Virus name']]

def remove_non_val_dates(dates):
    """removes no valid dates from the list of string dates with format 0000-00-00"""
    valid_dates = []
    count = 0
    for date in dates:
        if len(date) == 10:
            try:
                valid = pd.to_datetime(date)
                valid_dates.append(date)
            except:
                count = count + 1
                continue
    # print(count, " non valid dates")
    return valid_dates

def generate_datapoints(dates, mode, time_period, start_delay):
    """from the start date (+ start_delay), it calculates all the dates 'time_period' number of days apart from
    each other, and converts it to a list of numbers if it is datetime"""
    dates = np.sort(dates)
    start = dates[0]
    end = dates[len(dates)-1]
    timepoints = []
    if (mode == 'date'):
        start = pd.to_datetime(start)
        date = start + np.timedelta64(start_delay,'D')
    else:
        date = start + start_delay
    timepoints.append(start)
    while True:
        if (mode == 'date'):
            date =  date + np.timedelta64(time_period,'D')
        else:
            date = date + time_period
        timepoints.append(date)
        if(date >= end):
            break
    dates = timepoints
    if (mode == 'date'):
        timepoints = date2num(timepoints)
    return timepoints, dates


#!/usr/bin/env python
"""
Reduce samples that have too high energies by comparing
between the same group of samples.
The group is defined according to the name of directory before the last 5 digits.
For example, the directory `smpl_XX_YYYYYY_#####` where `#####` is the
last 5 digits and the group name would be `smpl_XX_YYYYY`.

Usage:
  reduce_high_energy_samples.py [options] DIRS...

Options:
  -h,--help  Show this message and exit.
  -o OUT     Output file name. [default: out.high_energy_samples]
  --threshold=THRESHOLD
             Threshold of energy/atom that determines high energy samples.
             [default: 1.0]
"""
from __future__ import print_function

import os,sys
from docopt import docopt
from datetime import datetime

from nappy.napsys import NAPSystem

__author__ = "RYO KOBAYASHI"
__version__ = "160727"

def get_obsolete_dirname():
    prefix = "obsolete_"
    today = datetime.today()
    return prefix+today.strftime("%y%m%d")

def get_groups(smpldirs):
    groups = {}
    ns = len(smpldirs)
    if ns < 100:
        ms = 1
    else:
        ms = ns/100
    for i,s in enumerate(smpldirs):
        if i%ms == 0:
            print('.',end=".")
        try:
            with open(s+'/erg.ref','r') as f:
                erg = float(f.readline())
        except:
            print('Failed to read erg.ref, so skip '+s)
            continue
        key = s[:-6]
        if not key in groups:
            groups[key] = []
        groups[key].append([s,erg])
    print('')
    return groups

def get_list_high_energy(gsmpls,threshold):
    emin = 1e+30
    highsmpls = []
    ergs = []
    for i,s in enumerate(gsmpls):
        smpldir = s[0]
        erg = s[1]
        #atoms = read(smpldir+'/POSCAR',format='vasp')
        atoms = NAPSystem(fname=smpldir+"/pos",format='pmd')
        natm = atoms.num_atoms()
        erg /= natm
        ergs.append(erg)
        emin = min(erg,emin)
    for i,s in enumerate(gsmpls):
        smpldir = s[0]
        erg = ergs[i]
        if erg-emin > threshold:
            highsmpls.append(smpldir)
    return highsmpls

if __name__ == "__main__":

    args = docopt(__doc__)
    smpldirs = args['DIRS']
    outfname = args['-o']
    threshold = float(args['--threshold'])

    print('grouping samples...')
    groups = get_groups(smpldirs)

    print('looking for high-energy samples...')
    highsmpls = []
    for g,smpls in groups.items():
        print('.',end='')
        highsmpls.extend(get_list_high_energy(smpls,threshold))
    print('')

    with open(outfname,'w') as f:
        for s in highsmpls:
            f.write(s+'\n')
    print('number of samples to be reduced = ',len(highsmpls))
    print('check '+outfname+' and run the following commands:')
    print('')
    # obsdir = get_obsolete_dirname()
    # print('  mkdir '+obsdir)
    # print('  for d in `cat '+outfname+'`; do mv $d '+obsdir
    #       +'/; done')
    # print('')


from typing import List

from fastapi import APIRouter, BackgroundTasks, HTTPException
from starlette.responses import JSONResponse
from tortoise.exceptions import DoesNotExist

from ..schema import plant
from ..utils.tasks import check_if_job_exists, scrape_for_plant
from ..utils import search

router = APIRouter()

@router.get(
    "/plant",
    tags=["plant"],
    response_model=List[plant.PlantOut_Pydantic],
    responses={403: {"detail": "Not authorized"}},
)
async def get_plants():
    return await plant.get_plants()

@router.get(
    "/plant/{latin_name}",
    tags=["plant"],
    response_model=plant.PlantOut_Pydantic,
    responses={
        404: {"detail": "Object does not exist"},
        403: {"detail": "Not authorized"},
    },
)
async def get_plant(
    latin_name: str,
    background_tasks: BackgroundTasks,
):
    try:
        return await plant.get_plant(latin_name)
    except DoesNotExist:
        if not await check_if_job_exists(latin_name):
            background_tasks.add_task(scrape_for_plant, latin_name)
            return JSONResponse(
                status_code=404, content={"detail": "Object does not exist"}
            )
        else:
            return await plant.get_plant(latin_name)

@router.post(
    "/plant",
    tags=["plant"],
    response_model=plant.PlantOut_Pydantic,
    responses={403: {"detail": "Not authorized"}},
)
async def create_plant(plant_in: plant.PlantIn_Pydantic):
    return await plant.create_plant(plant_in)

@router.put(
    "/plant/{latin_name}",
    tags=["plant"],
    response_model=plant.PlantOut_Pydantic,
    responses={403: {"detail": "Not authorized"}},
)
async def update_plant(latin_name: str, plant_in: plant.Plant_Update_Pydantic):
    p = await plant.update_plant(latin_name, plant_in)
    search.update_or_add_plant(p)
    return p


import os
import sys
# import time
# import configparser
import sqlite3
import hashlib
import logging
from optparse import OptionParser
from sync import scan_dir
from sync import parse_configs
from sync import db_create_tables
from sync import db_remove_deleted

logging.basicConfig(level=logging.INFO, format=' %(asctime)s - %(levelname)s - %(message)s')

# Global variable
config = {}
conn = None  # DB handle

# Global constant
CONFIG_FILE = 'data' + os.sep + 'sync.cfg'

# parms
parm = {'delete': False, 'scan': False}

class File(object):
    def __init__(self, file_name, file_md5, file_mtime, file_size, dir_name, root_dir, rel_path):
        self.file_name = file_name
        self.file_md5 = file_md5
        self.file_mtime = file_mtime
        self.file_size = file_size
        self.dir_name = dir_name
        self.root_dir = root_dir
        self.rel_path = rel_path

def parse_options():
    # Use optparse to get parms
    usage = "usage: %prog [options] scope_dir keep_dir"
    parser = OptionParser(usage=usage)
    parser.add_option("-d", "--delete", dest="delete", action="store_true", default=False,
                      help="Delete the duplicate files that are not in the keep directory.")
    parser.add_option("-s", "--scan", dest="scan", action="store_true", default=False,
                      help="Scan the scope directory. The default is to use the existing DB.")
    (options, args) = parser.parse_args()
    return options, args  # options: copy, rejects; args: source_dir

def db_get_name(dir_name):
    """
    Build a hash value from the concatenation of source_dir
    :param dir_name: valeur à hasher pour faire le db_name
    """
    m = hashlib.md5()
    m.update(dir_name.encode('utf-8'))
    db_name = m.hexdigest() + ".db"
    return db_name

def list_dup(db_h, root_dir):

    sel_md5 = \
        '''
        select file_md5, count(*)
          from file
         where file_md5 in (select file_md5 from file where dir_name = ?)
         group by file_md5
        having count(*) > 1
        '''
    sel_dup = \
        '''
        select dir_name, file_name, file_size, file_mtime
          from file
         where file_md5 = ?
       '''

    try:
        cur_md5 = db_h.cursor()
        cur_dup = db_h.cursor()
        for row_md5 in cur_md5.execute(sel_md5, [root_dir]):
            file_md5 = row_md5[0]
            logging.info("Possible duplicates: %s" % file_md5)
            for row_dup in cur_dup.execute(sel_dup, [file_md5]):
                dir_name = row_dup[0]
                file_name = row_dup[1]
                file_size = row_dup[2]
                file_mtime = row_dup[3]
                logging.info("    Fichier......: %s" % dir_name + os.sep + file_name)
                logging.info("        Size.....: %i" % file_size)
                logging.info("        MTime....: %s" % file_mtime)
            logging.info(" ")
    except sqlite3.Error as x:
        logging.error("SQL Error: \n" + str(x))

def delete_dup(db_h, keep_dir):
    sel_md5 = \
        '''
        select file_md5, count(*)
          from file
         where file_md5 in (select file_md5 from file where dir_name = ?)
         group by file_md5
        having count(*) > 1
        '''
    sel_dup = \
        '''
        select dir_name, file_name
          from file
         where file_md5 = ?
           and dir_name <> ?
        '''
    del_dup = \
        '''
        delete from file
         where file_md5 = ?
           and dir_name <> ?
       '''

    try:
        cur_md5 = db_h.cursor()
        cur_dup = db_h.cursor()
        for row_md5 in cur_md5.execute(sel_md5, [keep_dir]):
            file_md5 = row_md5[0]
            logging.info("Possible duplicates: %s" % file_md5)
            for row_dup in cur_dup.execute(sel_dup, [file_md5, keep_dir]):
                dir_name = row_dup[0]
                file_name = row_dup[1]
                logging.info("    Ce fichier sera effacé: %s" % dir_name + os.sep + file_name)
                if os.path.isfile(keep_dir + os.sep + file_name):
                    os.remove(dir_name + os.sep + file_name)
                else:
                    logging.error("    Ce fichier ne peut pas être effacé. " +
                                  " Il n'existe pas dans le répertoire à conserver.")
            logging.info(" ")
            cur_dup.execute(del_dup, [file_md5, keep_dir])

    except sqlite3.Error as x:
        logging.error("SQL Error: \n" + str(x))

def main():
    global parm, config, conn
    logging.info('Début du programme ' + sys.argv[0])

    # Get parameters and validate them
    (options, args) = parse_options()
    parm["delete"] = options.delete
    parm["scan"] = options.scan
    if len(args) < 2:
        logging.error("Ce programme a besoin de deux arguments, le dossier source et le dossier à protéger.")
        return 8
    source_dir = args[0]
    protec_dir = args[1]
    if source_dir.endswith(os.sep):
        source_dir = source_dir[0:-1]
    if protec_dir.endswith(os.sep):
        protec_dir = protec_dir[0:-1]

    logging.info("Paramètres:")
    logging.info("    Dossier scope................................: %s" % source_dir)
    logging.info("    Dossier à protéger...........................: %s" % protec_dir)
    if not os.path.isdir(source_dir):
        logging.error("Le dossier source n'existe pas.")
        return 8
    if not os.path.isdir(protec_dir):
        logging.error("Le dossier à protéger n'existe pas.")
        return 8

    if parm["delete"]:
        logging.info("    Option de suppression........................: Oui")
    else:
        logging.info("    Option de suppression........................: Non")
    if parm["scan"]:
        logging.info("    Option de population de la BD................: Oui")
    else:
        logging.info("    Option de population de la BD................: Non")
    logging.info(" ")

    # Read config file
    config = parse_configs()
    logging.info("Configurations:")
    logging.info("    Fichier de configuration.....................: " + CONFIG_FILE)
    for ext in config['accept_list']:
        logging.info("    Accepted extension...........................: %s" % ext)
    for ext in config['reject_list']:
        logging.info("    Rejected extension...........................: %s" % ext)

    db_name = db_get_name(source_dir)
    db_path = source_dir + os.sep + db_name
    logging.info("    Database file................................: " + db_name)
    logging.info(" ")

    conn = sqlite3.connect(db_path)
    db_create_tables(conn)            # Create the DB objects
    if parm['scan']:
        scan_dir(conn, source_dir)    # Create inventory of the files in the source directory structure
    # Pu sûr que c'est une bonne idée...
    # if parm['delete']:
    #    delete_dup(conn, protec_dir)
    #    db_remove_deleted(conn)       # Remove deleted files from db
    # else:
    list_dup(conn, source_dir)

    conn.commit()
    conn.close()

    logging.info('Fin du programme ' + sys.argv[0])
    return 0

if __name__ == "__main__":
    sys.exit(main())


import praw
import configparser
import time
import datetime
from praw.models import MoreComments

cp = configparser.RawConfigParser()
configFilePath = './client.config'
cp.read(configFilePath)

current_client_id = cp.get('client-config', 'client_id')
current_client_secret = cp.get('client-config', 'client_secret')

reddit = praw.Reddit(
    user_agent="Comment Extraction (by u/chriskok1337)",
    client_id=current_client_id,
    client_secret=current_client_secret
)

# Time to rest between reddit random post requests
TIME_BETWEEN_REQUESTS = 2.1

"""Gets and returns formatted subreddit metadata

    Args:
        subreddit (str): the subreddit we want to get metadata for

    Returns:
        subreddit_dict: a dictionary of relevant metadata for the subreddit
"""
def get_subreddit_meta(subreddit):
    subreddit_dict = {}
    subreddit = reddit.subreddit(subreddit)

    subreddit_dict['id'] = subreddit.id
    subreddit_dict['created_utc'] = subreddit.created_utc
    subreddit_dict['year'] = datetime.date.fromtimestamp(subreddit.created_utc).year
    subreddit_dict['description'] = subreddit.description
    subreddit_dict['display_name'] = subreddit.display_name
    subreddit_dict['name'] = subreddit.name
    subreddit_dict['subscribers'] = subreddit.subscribers
    subreddit_dict['public_description'] = subreddit.public_description

    return subreddit_dict

"""Extracts and processes relevant information for a submission

    Args:
        sub (Submission): the submission we're extracting data from
        comment_sort (str): sorting order for comments [confidence, controversial, new, old, q&a, top]

    Returns:
        sub_dict: a dictionary of relevant information for this submission
"""
def save_submission(sub, comment_sort='top'):
    sub_dict = {}
    sub_dict['id'] = sub.id
    sub_dict['num_comments'] = sub.num_comments
    sub_dict['score'] = sub.score
    sub_dict['selftext'] = sub.selftext
    sub_dict['stickied'] = sub.stickied
    sub_dict['title'] = sub.title
    sub_dict['upvote_ratio'] = sub.upvote_ratio
    sub_dict['created_utc'] = sub.created_utc
    sub_dict['year'] = datetime.date.fromtimestamp(sub.created_utc).year

    current_top_comments = []
    # Set comment sort to best before retrieving comments
    sub.comment_sort = comment_sort
    # Limit to, at most, 10 top level comments
    sub.comment_limit = 10
    for top_level_comment in sub.comments:
        if isinstance(top_level_comment, MoreComments):
            continue
        current_top_comments.append(top_level_comment.body)

    sub_dict['top_comments'] = current_top_comments

    return sub_dict

"""Gets and returns formatted TOP posts (and related comments) by subreddit

    Args:
        subreddit (str): the subreddit we want to scrape posts for
        limit (int): limit to the number of submissions
        time_range (str): how far back to look for gathering data [all, day, hour, month, week, year]

    Returns:
        post_dict: a dictionary of subreddit submission dictionaries. ID as the key and relevant data stored as the value.
"""
def get_top_posts(subreddit, limit=None, time_range="all"):
    submissions = reddit.subreddit(subreddit).top(time_range, limit=limit)
    post_dict = {}
    for sub in submissions:
        current_ID = sub.id
        sub_dict = save_submission(sub)
        post_dict[current_ID] = sub_dict

    return post_dict

"""Gets and returns formatted CONTROVERSIAL posts (and related comments) by subreddit

    Args:
        subreddit (str): the subreddit we want to scrape posts for
        limit (int): limit to the number of submissions
        time_range (str): how far back to look for gathering data [all, day, hour, month, week, year]

    Returns:
        post_dict: a dictionary of subreddit submission dictionaries. ID as the key and relevant data stored as the value.
"""
def get_controversial_posts(subreddit, limit=None, time_range="all"):
    submissions = reddit.subreddit(subreddit).controversial(time_range, limit=limit)
    post_dict = {}
    for sub in submissions:
        current_ID = sub.id
        sub_dict = save_submission(sub)
        post_dict[current_ID] = sub_dict

    return post_dict

"""Gets and returns formatted HOT posts (and related comments) by subreddit

    Args:
        subreddit (str): the subreddit we want to scrape posts for
        limit (int): limit to the number of submissions
        time_range (str): how far back to look for gathering data [all, day, hour, month, week, year]

    Returns:
        post_dict: a dictionary of subreddit submission dictionaries. ID as the key and relevant data stored as the value.
"""
def get_hot_posts(subreddit, limit=None):
    submissions = reddit.subreddit(subreddit).hot(limit=limit)
    post_dict = {}
    for sub in submissions:
        current_ID = sub.id
        sub_dict = save_submission(sub)
        post_dict[current_ID] = sub_dict

    return post_dict

"""Gets and returns formatted RANDOM posts (and related comments) by subreddit

    Args:
        subreddit (str): the subreddit we want to scrape posts for
        limit (int): limit to the number of submissions

    Returns:
        post_dict: a dictionary of subreddit submission dictionaries. ID as the key and relevant data stored as the value.
"""
def get_random_posts(subreddit, limit=1000):
    post_dict = {}
    for i in range(limit):
        submission = reddit.subreddit(subreddit).random()
        if (submission is None):
            print('r/{} DOES NOT allow RANDOM scraping.'.format(subreddit))
            return post_dict
        sub_dict = save_submission(submission)
        current_ID = submission.id
        post_dict[current_ID] = sub_dict
        time.sleep(TIME_BETWEEN_REQUESTS)
        if (i % 100 == 0 and i > 0): print('{}/{} random posts scraped'.format(i, limit))

    return post_dict

# TODO: include functions to grab through random and search requests - will be able to give us posts not found above
# NOTE: its more difficult because random only returns one at a time so we might have to send requests
# and fill in as necessary, keeping in mind the 2 second request interval
# NOTE: search requests will be difficult too because we need to know what we're searching for
# NOTE: we have to consider this because the Reddit API no longer allows queries within a timeframe,
# plus it only allows for 1000 submissions per above queries.

def main():
    # post_dict = get_top_posts('learnpython', 1, 'month')
    # print(post_dict)
    # post_dict = get_controversial_posts('learnpython', 1)
    # print(post_dict)
    # post_dict = get_hot_posts('learnpython', 1)
    # print(post_dict)
    post_dict = get_random_posts('learnpython', 1)
    print(post_dict)

    # sub_dict = get_subreddit_meta('learnpython')
    # print(sub_dict)

if __name__ == "__main__":
    main()


from django.shortcuts import render
from django.http import HttpResponse, HttpResponseRedirect
#from django.template import loader
from django.utils import timezone

from .models import Item, UserList
from .forms import NameForm
from .list_form import UserListForm

def index(request):
    item_list = Item.objects.all()
    produce_items = Item.objects.filter(item_type="produce")
    meat_items = Item.objects.filter(item_type="meat_and_fish")
    p_goods = Item.objects.filter(item_type="packaged_goods")
    bulk = Item.objects.filter(item_type="bulk_goods")
    item_types = Item.ITEM_TYPES
    item_types_hr = [x[1] for x in Item.ITEM_TYPES]
    item_status_hr = [x[1] for x in Item.ITEM_STATUS]
    context = {
        'item_list': item_list,
        'item_desc': item_types_hr,
        'bulk': bulk,
        'p_goods': p_goods,
        'meat_items': meat_items,
        'produce_items': produce_items,
        'status_types': item_status_hr,
        'item_types': item_types_hr
    }
    return render(request, 'grocery_list/index.html', context)

def list_display(request, list_id):
    mylist = UserList.objects.get(pk=list_id)
    return HttpResponse(mylist)

def detail(request, item_id):
    response = "Viewing details about %s."
    return HttpResponse(response % item_id)

def need(request):
    needed_item_list = Item.objects.exclude(status='good')
    context = {
        'needed_item_list': needed_item_list,
    }
    return render(request, 'grocery_list/get.html', context)

def get_list(request):
    if request.method == 'POST':
        form = UserListForm(request.POST)
        if form.is_valid():
            print(form.cleaned_data)
            new_list = form.save()
            return HttpResponseRedirect('/thanks/')
    else:
        form = UserListForm()

    return render(request, 'grocery_list/list.html', {'form': form})

def thanks(request):
    response = "Thanks for keeping things updated"
    return HttpResponse(response)


__author__ = 'techbk'

import asyncio
import aiohttp

@asyncio.coroutine
def fetch_page(client, url):
    response = yield from client.get(url)
    assert response.status == 200
    #result = yield from response.read()
    text = yield from response.text()
    return text

loop = asyncio.get_event_loop()
client = aiohttp.ClientSession(loop=loop)
content = loop.run_until_complete(fetch_page(client, 'https://api.github.com/events'))
print(content)
client.close()


import keras
from keras.models import load_model
from keras import backend as k
import os
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
import sys
import sklearn.metrics as metrics
import cv2
import sys
import re
from keras.utils import np_utils
from keras.preprocessing import image
from keras.applications.vgg16 import preprocess_input, decode_predictions
import csv
#to make tensorflow shut up
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)

#targeted fast gradient sign method, which pushes a class towards a specific label
def single_targeted_FGSM(model, img, label, epsilon):
	'''
	model : keras Sequential model
			neural network object to get info from
	imgs : images of 224 by 224 by 3
		the input images that we are going to alter
	targets : array
			one-hot encoding vectors that contains the targeted labels of the classes
	'''
	target = get_target_label(label)
	y_target = keras.Input(shape=target.shape)
	#define loss function
	loss = k.mean(k.categorical_crossentropy(y_target, model.output))
	#get derivative/gradient of loss wrt input
	loss_gradient = k.gradients(loss, model.inputs)
	#build function
	func = k.function(model.inputs + [y_target], loss_gradient)
	#get gradient
	gradient = func([img, target])
	gradient = np.sign(gradient)
	print(gradient)
	gradient = gradient * epsilon
	print(gradient)
	adv_img = img - gradient
	return adv_img

def test_imgs(model, imgs, labels):
	'''
	model: keras Sequential model object
		   model with which to make predictions
	imgs: numpy array
		  vector of images to test
	labels: numpy array
			one-hot labels of images, should have same length as imgs
	'''
	accuracy = 0
	avg_confidence = 0

	for i in range(0, len(labels)):
		x = imgs[i]
		#check to see if image is 4 dimensional
		if len(x.shape) != 4:
			#if not, wrap it in one dimension
			x = np.expand_dims(x, axis=0)
		#this is for VGG16 network only
		x = preprocess_input(x)
		#get prediction of image
		prediction = model.predict(x)
		#get class label
		pred = np.argmax(prediction[0])
		#get truth
		actual = np.argmax(labels[i])
		#get confidence of prediction
		confidence = max(prediction[0])
		avg_confidence += confidence
		#check for accuracy
		if pred == actual:
			accuracy += 1
	return accuracy/len(labels), avg_confidence/len(labels)

#function to nicely display images in matplotlib
def display_image(model, x, x_adv, label, e):
	'''
	model : keras Sequential model object
			the neural network trained on the CINIC 10 images
	x : 224 by 224 by 3 numpy array
		the unaltered, original image
	x_adv : 224 by 224 by 3 numpy array
			the adversarial image
	label : numpy array
			one-hot encoded vector, true labels of the images
	e : int or float
		how much the image was altered, used here to output stats
	'''
	#actual labels of the images
	class_labels = ["airplane", "automobile", "bird", "cat", "deer", "dog", "frog", "horse", "ship", "truck"]

	#initialize plot
	f = plt.figure()
	plt.title("Class %s example" % class_labels[np.argmax(label)])
	plt.axis("off")

	#add original image
	f.add_subplot(1, 2, 1)
	plt.title("Original Image")
	#prediction and confidence
	p, c = prediction(model, x)
	#some preprocessing before sending image off to matplotlib
	x = np.squeeze(x)
	#needed to reverse channels from RGB to BGR so the colors didnt come out blue in matplotlib
	plt.imshow(x[...,::-1])
	#show annotation with prediction and confidence score
	plt.annotate("Original Image\nClass prediction: %s\nProbability: %.4f" % (class_labels[p], c), (0, 0), (0, -20), xycoords='axes fraction', textcoords='offset points', va='top')

	#add adversarial image
	f.add_subplot(1, 2, 2)
	plt.title("Adversarial Image, e=%d" % e)
	p, c = prediction(model, x_adv)
	#some preprocessing before sending image off to matplotlib
	x_adv = np.clip(x_adv, 0, 255)
	x_adv = np.squeeze(x_adv)
	x_adv /= 255
	plt.imshow(x_adv[...,::-1])
	#show annotation with prediction and confidence score
	plt.annotate("Adversarial Image\nClass prediction: %s\nProbability: %.4f" % (class_labels[p], c), (0, 0), (0, -20), xycoords='axes fraction', textcoords='offset points', va='top')
	plt.savefig("results/img_%d.png" % e, bbox_inches="tight")

#helper function to predict image class in img_test function
def prediction(model, image):
	'''
	model : keras Sequential model
			neural network object to prediction with
	img : image of 224 by 224 by 3
		the image to predict
	'''
    #reshape image for keras
	img = image.reshape(1, 224, 224, 3)
    #make prediction
	pred = model.predict(img)
	#flatten
	pred = list(pred[0])
	#get index of class label
	p = pred.index(max(pred))
	#get confidence score
	c = max(pred)
	#return prediction label and confidence score
	return p, c

#regular fast gradient sign method
def FGSM(model, imgs, labels):
	'''
	model : keras Sequential model
			neural network object to get info from
	imgs : image of 224 by 224 by 3
		the input image that we are going to alter
	labels : array
			one-hot encoding vector that contains the true label of the class
	'''
	#get shape of label to use later
	y_true = keras.Input(shape=labels[0].shape)
	#define loss function
	loss = k.mean(k.categorical_crossentropy(y_true, model.output))
	#get derivative/gradient of loss wrt input
	loss_gradient = k.gradients(loss, model.inputs)
	#build function
	func = k.function(model.inputs + [y_true], loss_gradient)
	#get gradients
	gradients = []
	start = 0
	end = int(len(labels)/20)
	increment = int(len(labels)/20)
	#loop to avoid OOM error
	for i in range(0, 20):
		tmp = func([np.squeeze(imgs[start:end]), labels[start:end]])[0]
		start = end
		end += increment
		gradients.append(tmp)
	gradients = np.array(gradients)
	gradients = np.reshape(gradients, (len(labels), 224, 224, 3))
	return gradients

#targeted fast gradient sign method, which pushes a class towards a specific label
def targeted_FGSM(model, imgs, targets):
	'''
	model : keras Sequential model
			neural network object to get info from
	imgs : images of 224 by 224 by 3
		the input images that we are going to alter
	targets : array
			one-hot encoding vectors that contains the targeted labels of the classes
	'''
	y_target = keras.Input(shape=targets[0].shape)
	#define loss function
	loss = k.mean(k.categorical_crossentropy(y_target, model.output))
	#get derivative/gradient of loss wrt input
	loss_gradient = k.gradients(loss, model.inputs)
	#build function
	func = k.function(model.inputs + [y_target], loss_gradient)
	#get gradients
	gradients = []
	start = 0
	end = int(len(targets)/123)
	increment = int(len(targets)/123)
	#loop to avoid OOM error
	for i in range(0, 123):
		tmp = func([np.squeeze(imgs[start:end]), targets[start:end]])[0]
		start = end
		end += increment
		gradients.append(tmp)
	gradients = np.array(gradients)
	print(gradients.shape)
	gradients = np.reshape(gradients, (len(targets), 224, 224, 3))
	return gradients

#helper method to make a target label
def get_target_label(label):
	'''
	label : one-hot vector
			true label of an image
	'''
	true_class = np.argmax(label)
	#push index up by three labels, wrap around in case out of bounds
	target_class = (true_class + 3) % 10
	#convert to one hot vector
	one_hot = np.zeros(10)
	one_hot[target_class] = 1
	return one_hot

def add_imgs(imgs, gradients, epsilon):
	'''
	imgs : numpy array
		   array of original images
	gradients : numpy array
			   array of gradients obtained from fgsm or targeted_fgsm
	epsilon : integer or float
			  number by which to alter the pixel intensities
	'''
	#initialize empty numpy array
	adv_imgs = []
	#create adversarial images here
	for i in range(0, len(imgs)):
		adv = imgs[i] + (np.sign(gradients[i]) * epsilon)
		adv = np.clip(adv, 0, 255)
		adv_imgs.append(adv)
	return adv_imgs

# don't run this again
def fix_channels(name):
	imgs = np.load("data/%s_test_set.npy" % name)
	for i in range(0, len(imgs)):
		imgs[i] = np.clip(imgs[i], 0, 255)
	np.save("data/%s_test_set.npy" % name, imgs)

if __name__ == "__main__":

	model_list = ["subsample", "An Diff", "Med Gauss", "Nonlocal means", "ROF", "TVD"]

	# imgs = np.load("data/%d_test_set.npy" % model_list[i])
	imgs = np.load("data/subsample_test_set.npy")
	print("Loaded in %d images" % len(imgs))
	t_labels = np.load("data/subsample_test_labels.npy")
	# print("Loaded in %d labels" % len(t_labels))
	print("Loaded in %d labels" % len(t_labels))
	labels = np_utils.to_categorical(t_labels, 10)

	base_test = True

	if not base_test:
		for i in range(1, len(model_list)):
			model = load_model("models/%s_vgg16_10_HNN.h5" % model_list[i])
			print("Loaded in %s model" % model_list[i])

			gradients = None
			if not os.path.exists("data/gradients_HNN_%s_filtered_BUG.npy" % model_list[i]):
				print("Did not find %s gradients, creating them now" % model_list[i])
				gradients = FGSM(model, imgs, labels)
				np.save("data/gradients_HNN_%s_filtered_BUG.npy" % model_list[i], gradients)
				print("Made %s gradients" % model_list[i])
			else:
				print("Loaded in %s gradients" % model_list[i])
				gradients = np.load("data/gradients_HNN_%s_filtered.npy" % model_list[i])

			for epsilon in range(1, 11):
				print("Adding e = %d" % epsilon)
				x_adv = add_imgs(imgs, gradients, epsilon)
				if not os.path.exists("FGSM_imgs/og_img_filter_network/%s_BUG" % model_list[i]):
					os.mkdir("FGSM_imgs/og_img_filter_network/%s_BUG" % model_list[i])
				np.save("FGSM_imgs/og_img_filter_network/%s_BUG/adversarial_images_e=%d.npy" % (model_list[i], epsilon), x_adv)
				print("Testing e = %d" % epsilon)
				acc, conf = test_imgs(model, x_adv, labels)
				print("Accuracy: %.4f" % acc)
				print("Confidence: %.4f" % conf)
				del x_adv
			del gradients

			print("Completed %s" % model_list[i])
			sys.exit(0)

	else:
		for i in range(0, len(model_list)):
			acc = []
			method = model_list[i]
			model = load_model("models/%s_vgg16_10_HNN.h5" % method)
			print(method)
			t_acc, conf = test_imgs(model, imgs, labels)
			print(t_acc)
			acc.append(t_acc)
			for epsilon in range(1, 11):
					print("Testing e = %d" % epsilon)
					adv = np.squeeze(np.load("FGSM_imgs/og_img_filter_network/%s/adversarial_images_e=%d.npy" % (method, epsilon)))
					#if (i == 10):
					#	plt.imshow(adv[1])
					t_acc, conf = test_imgs(model, adv, labels)
					print(t_acc)
					acc.append(t_acc)
		np.save("whitebox/clean/before_overall_%s.npy" % method, acc)
		sys.exit(0)


import cv2
import numpy as np
import scipy.interpolate

def createCurveFunc(points):
    """Return a function derived from control points"""
    if points is None:
        return None
    numPoints = len(points)
    if numPoints < 2:
        return None

    xs,ys = zip(*points)

    if numPoints < 4:
        kind = 'linear' # 'quadratic is not implemented'

    else:
        kind = 'cubic'
    return scipy.interpolate.interp1d(xs,ys,kind,bounds_error=False)
    # bounds_error = false to permit extrapolation and interpolation

# print(createCurveFunc([(1,3),(3,4),(4,7)]))

def createLookupArray(func, length=256):
    """
    returns a lookup for whole-number inputs to a function
    the lookup values are clamped to [0,length-1]
    """
    if func is None:
        return None

    lookupArray = np.empty(length)
    i=0
    while i <length:
        func_i = func(i)
        lookupArray[i] = min(max(0,func_i),length-1)
        i+=1
    return lookupArray

def applyLookupArray(lookupArray, src,dest):
    """ map a source to destination using lookup"""
    if lookupArray is None:
        return
    dest[:] = lookupArray[src]

# combining two curve function into one function before creating a lookup array (optimization)

def createCompositefuntion(func0, func1):
    """ Returns a composite of two functions."""
    if func0 is None:
        return func1
    if func1 is None:
        return func0
    return lambda x: func0(func1(x))

# for image with multiple channel, instead of splitting and merging, flatten and make it 1-d and pass to create lookup array

def createFlatView(array):
    """ Returns 1-d view of an array of any dimension """
    flatView = array.view()
    flatView.shape = array.size # size will return the mxn value of the array
    return flatView

def isGray(image):
    """return true if the image has one channel per pixel"""
    return image.ndim<3

def widthHeightDividedBy(image, divisor):
    """returns and image's dimension divided by number"""
    h,w = image.shape[:2]
    # tup = []
    # # print(type(h))
    # # h = (h/divisor).astype(int)
    # # w = (w/divisor).astype(int)
    # for i in range(len(h)):
    #     tup.append(tuple((w[i],h[i])))

    # print(tup)
    # print(w)
    # print(type(w))
    # print(h)
    # concat = np.concatenate((w,h))
    return (int(w/divisor), int(h/divisor))


from datetime import datetime
from airflow import DAG
from airflow.operators.bash import BashOperator
from airflow.sensors.external_task import ExternalTaskSensor

with DAG(dag_id="7.2-externalTaskSensor",
         description="DAG Secundario",
         schedule_interval="@daily",
         start_date=datetime(2022, 5, 1),
         end_date=datetime(2022, 8, 1),
         max_active_runs=1) as dag:

    t1 = ExternalTaskSensor(task_id="waiting_dag",
                            external_dag_id="7.1-externalTaskSensor",
                            external_task_id="tarea1",
                            poke_interval=10)

    t2 = BashOperator(task_id="tarea2",
                      bash_command="sleep 10 && echo 'DAG 2 finalizado!'",
                      depends_on_past=True)

    t1 >> t2


# -*- coding:utf-8 -*-
"""
json的用法：就是调用json.dump()和json.load()方法，
dump()方法是把获取的数据放进去，load()方法是把获取的数据拿出来
调用这两个方法，都需要传递两个参数，一个是需要传的那个，另一个是传到哪里的参数
"""
import json
username = input("What is your name? ")
filename = 'username.json'
with open(filename, 'w') as item:
    '''
    这里调用的dump方法就传递了两个参数，一个是用户输入的参数username，
    另一个item就是filename的别名，也就是把这个username参数传递到username.json文件中，做一个参数储存
    还有就是load()方法了，load方法用来把json中的数据拿出来，但是这个函数的参数可以只传一个，
    那就是 username = json.load(item)，这种，把整个文件中的数据全部拿出来
    '''
    json.dump(username, item)
    print("We'll remember you when you come back, " + username + "!")

用json保存用户输入信息.py

用json保存用户输入信息.py


import tools.sapi as sapi

def config():
    url = input('Link?\n')
    li_at = ''
    li_a = ''
    jsessionid = ''
    file_name = input('File name?\n')

    sapi.api(url, li_at, li_a, jsessionid, file_name)

config()


import streamlit as st
# from create import create
from create import create_player
from create import create_inventory
from create import create_coach
# from database import create_table
from delete import delete
from read import read_team
from read import read_player
from update import update

def main():
    st.title("Indian Premium League D-Base")
    menu = ["Add Player","Add inventory","Add coach", "View PLAYER", "View TEAM", "Edit MEDIC", "Remove PLAYER"]
    choice = st.sidebar.selectbox("Menu", menu)
    # create_table()
    if choice == "Add Player":
        st.subheader("Enter PLAYER Details:")
        create_player()
    elif choice == "Add inventory":
        st.subheader("Enter INVENTORY Details:")
        create_inventory()
    elif choice == "Add coach":
        st.subheader("Enter COACH Details:")
        create_coach()
    elif choice == "View PLAYER":
        st.subheader("View created players")
        read_player()
    elif choice == "View TEAM":
        st.subheader("View created teams")
        read_team()
    elif choice == "Edit MEDIC":
        st.subheader("Update the medic hospital")
        update()
    elif choice == "Remove PLAYER":
        st.subheader("Delete players")
        delete()
    else:
        st.subheader("Divided by teams, United by king of good times.")
if __name__ == '__main__':
    main()


import pandas as pd

from sklearn.model_selection import train_test_split

def read():
    #HDFS_DATASET = "data/hdfs/anomaly/prepared.csv"
    FIREWALL_DATASET = "data/firewall/anomaly/prepared.csv"
    dataset = pd.read_csv(FIREWALL_DATASET)
    logs=dataset['log'].tolist()
    print(len(logs))

def main():
    HDFS_DATASET = "data/hdfs/anomaly/prepared.csv"
    FIREWALL_DATASET = "data/firewall/anomaly/prepared.csv"
    fdataset = pd.read_csv(FIREWALL_DATASET)
    fdataset.drop(['type'], axis=1)
    hdataset = pd.read_csv(HDFS_DATASET)
    dataset = pd.concat([fdataset,hdataset])
    logs=dataset['log'].tolist()
    labels=dataset['label'].tolist()
    #types=dataset['type'].tolist()
    # DO NOT FORGET to change stratify column if ..
    X_train, X_test, y_train, y_test = train_test_split(logs, labels,
                                                        train_size=0.80,
                                                        random_state=666,
                                                        stratify=labels)

    train_dict={}
    train_dict['log'] = X_train
    train_dict['label'] = y_train

    train_pd = pd.DataFrame(train_dict)
    train_pd.to_csv("data/all/anomaly/train.csv", index=False, header=True)

    test_dict={}
    test_dict['log'] = X_test
    test_dict['label'] = y_test

    test_pd = pd.DataFrame(test_dict)
    test_pd.to_csv("data/all/anomaly/eval.csv", index=False, header=True)

if __name__ == "__main__":
    main()


import unittest
import sys

from robot.variables import variables, is_list_var, is_scalar_var, is_var
from robot.errors import *
from robot import utils
from robot.utils.asserts import *

SCALARS = [ '${var}', '${  v A  R }' ]
LISTS = [ '@{var}', '@{  v A  R }' ]
NOKS = [ 'var', '$var', '${var', '${va}r', '@{va}r', '@var', '%{var}',
         ' ${var}', '@{var} ', '\\${var}', '\\\\${var}' ]

# Simple objects needed when testing assigning objects to variables.
# JavaObject lives in '../../acceptance/testdata/libraries'

class PythonObject:
    def __init__(self, a, b):
        self.a = a
        self.b = b
    def __str__(self):
        return '(%s, %s)' % (self.a, self.b)
    __repr__ = __str__

if utils.is_jython:
    import JavaObject

class TestIsMethods(unittest.TestCase):

    def test_is_var(self):
        for ok in SCALARS + LISTS:
            assert is_var(ok)
        for nok in NOKS:
            assert not is_var(nok)

    def test_is_scalar_var(self):
        for ok in SCALARS:
            assert is_scalar_var(ok)
        for nok in LISTS + NOKS:
            assert not is_scalar_var(nok)

    def test_is_list_var(self):
        for ok in LISTS:
            assert is_list_var(ok)
        for nok in SCALARS + NOKS:
            assert not is_list_var(nok)

class TestVariables(unittest.TestCase):

    def setUp(self):
        self.varz = variables.Variables()

    def test_set(self):
        for var in SCALARS + LISTS:
            assert not self.varz.has_key(var)
            self.varz[var] = ['value']
            assert self.varz.has_key(var), var
            assert self.varz.has_key(var.lower().replace(' ',''))
            self.varz.clear()

    def test_set_invalid(self):
        for var in NOKS:
            try:
                self.varz[var] = ['value']
            except DataError:
                pass
            else:
                fail()

    def test_set_scalar(self):
        for var in SCALARS:
            for value in [ 'str', 10, ['hi','u'], ['hi',2], {'a':1,'b':2}, self,
                           None, unittest.TestCase ]:
                self.varz[var] = value
                act = self.varz[var]
                assert act == value, '%s: %s %s != %s %s' \
                        % (var, act, type(act), value, type(value))
                self.varz.clear()
        self.varz['${myvar}'] = ''
        assert_equals(self.varz['${myvar}'], '')

    def test_update(self):
        self.varz['${a}'] = 1
        self.varz.update({'${b}':2})
        for k, v in [('${a}', 1), ('${b}', 2)]:
            assert_true(k in self.varz)
            assert_true(k in self.varz.keys())
            assert_equals(self.varz[k], v)

    def test_update_invalid(self):
        self.varz['${a}'] = 1
        assert_raises(DataError, self.varz.update, {'invalid variable name':2})

    def test_set_list(self):
        for var in LISTS:
            for value in [ [], [''], ['str'], [10], ['hi','u'], ['hi',2],
                           [{'a':1,'b':2}, self, None] ]:
                self.varz[var] = value
                assert_equals(self.varz[var], value)
                self.varz.clear()

    def test_getitem_invalid(self):
        for var in NOKS:
            self.assertRaises(DataError, self.varz.__getitem__, var)

    def test_has_key(self):
        self.varz['${k}'] = 'v'
        assert self.varz.has_key('${k}')
        assert self.varz.has_key('${1}')
        assert self.varz.has_key('${k.upper()}')
        assert not self.varz.has_key('${non-existing}')

    def test_contains(self):
        self.varz['${k}'] = 'v'
        assert '${k}' in self.varz
        assert '${-3}' in self.varz
        assert '${k.upper()}' in self.varz
        assert '${nok}' not in self.varz

    def test_replace_scalar(self):
        self.varz['${foo}'] = 'bar'
        self.varz['${a}'] = 'ari'
        for inp, exp in [ ('${foo}','bar'), ('${a}','ari'),
                          ('${a','${a'), ('',''), ('hii','hii'),
                          ("Let's go to ${foo}!", "Let's go to bar!"),
                          ('${foo}ba${a}-${a}', 'barbaari-ari') ]:
            assert_equals(self.varz.replace_scalar(inp), exp)

    def test_replace_list(self):
        self.varz['@{L}'] = ['v1','v2']
        self.varz['@{E}'] = []
        self.varz['@{S}'] = ['1','2','3']
        for inp, exp in [ (['@{L}'], ['v1','v2']),
                          (['@{L}','v3'], ['v1','v2','v3']),
                          (['v0','@{L}','@{E}','v@{S}[2]'], ['v0','v1','v2','v3']),
                          ([], []), (['hi u','hi 2',3], ['hi u','hi 2',3]) ]:
            assert_equals(self.varz.replace_list(inp), exp)

    def test_replace_list_in_scalar_context(self):
        self.varz['@{list}'] = ['v1','v2']
        assert_equals(self.varz.replace_list(['@{list}']), ['v1', 'v2'])
        assert_equals(self.varz.replace_list(['-@{list}-']), ["-['v1', 'v2']-"])

    def test_replace_list_item(self):
        self.varz['@{L}'] = ['v0','v1']
        assert_equal(self.varz.replace_list(['@{L}[0]']), ['v0'])
        assert_equal(self.varz.replace_scalar('@{L}[1]'), 'v1')
        assert_equal(self.varz.replace_scalar('-@{L}[0]@{L}[1]@{L}[0]-'), '-v0v1v0-')
        self.varz['@{L2}'] = ['v0',['v11','v12']]
        assert_equal(self.varz.replace_list(['@{L2}[0]']), ['v0'])
        assert_equal(self.varz.replace_list(['@{L2}[1]']), [['v11','v12']])
        assert_equal(self.varz.replace_scalar('@{L2}[0]'), 'v0')
        assert_equal(self.varz.replace_scalar('@{L2}[1]'), ['v11','v12'])
        assert_equal(self.varz.replace_list(['@{L}[0]','@{L2}[1]']), ['v0',['v11','v12']])

    def test_replace_non_strings(self):
        self.varz['${d}'] = {'a':1,'b':2}
        self.varz['${n}'] = None
        assert_equal(self.varz.replace_scalar('${d}'), {'a':1,'b':2})
        assert_equal(self.varz.replace_scalar('${n}'), None)

    def test_replace_non_strings_inside_string(self):
        class Example:
            def __str__(self):
                return 'Hello'
        self.varz['${h}'] = Example()
        self.varz['${w}'] = 'world'
        res = self.varz.replace_scalar('Another "${h} ${w}" example')
        assert_equals(res, 'Another "Hello world" example')

    def test_replace_list_item_invalid(self):
        self.varz['@{L}'] = ['v0','v1','v3']
        for inv in [ '@{L}[3]', '@{NON}[0]', '@{L[2]}' ]:
            self.assertRaises(DataError, self.varz.replace_list, [inv])

    def test_replace_non_existing_list(self):
        self.assertRaises(DataError, self.varz.replace_list, ['${nonexisting}'])

    def test_replace_non_existing_scalar(self):
        self.assertRaises(DataError, self.varz.replace_scalar, '${nonexisting}')

    def test_replace_non_existing_string(self):
        self.assertRaises(DataError, self.varz.replace_string, '${nonexisting}')

    def test_replace_escaped(self):
        self.varz['${foo}'] = 'bar'
        for inp, exp in [ (r'\${foo}', r'${foo}'),
                          (r'\\${foo}', r'\bar'),
                          (r'\\\${foo}', r'\${foo}'),
                          (r'\\\\${foo}', r'\\bar'),
                          (r'\\\\\${foo}', r'\\${foo}') ]:
            assert_equals(self.varz.replace_scalar(inp), exp)

    def test_variables_in_value(self):
        self.varz['${exists}'] = 'Variable exists but is still not replaced'
        self.varz['${test}'] = '${exists} & ${does_not_exist}'
        assert_equals(self.varz['${test}'], '${exists} & ${does_not_exist}')
        self.varz['@{test}'] = ['${exists}', '&', '${does_not_exist}']
        assert_equals(self.varz['@{test}'], '${exists} & ${does_not_exist}'.split())

    def test_variable_as_object(self):
        obj = PythonObject('a', 1)
        self.varz['${obj}'] = obj
        assert_equals(self.varz['${obj}'], obj)
        expected = ['Some text here %s and %s there' % (obj,obj)]
        actual = self.varz.replace_list(['Some text here ${obj} and ${obj} there'])
        assert_equals(actual, expected)

    def test_extended_variables(self):
        # Extended variables are vars like ${obj.name} when we have var ${obj}
        obj = PythonObject('a', [1,2,3])
        dic = { 'a': 1, 'o': obj }
        self.varz['${obj}'] = obj
        self.varz['${dic}'] = dic
        assert_equals(self.varz.replace_scalar('${obj.a}'), 'a')
        assert_equals(self.varz.replace_scalar('${obj.b}'), [1,2,3])
        assert_equals(self.varz.replace_scalar('${obj.b[0]}-${obj.b[1]}'), '1-2')
        assert_equals(self.varz.replace_scalar('${dic["a"]}'), 1)
        assert_equals(self.varz.replace_scalar('${dic["o"]}'), obj)
        assert_equals(self.varz.replace_scalar('-${dic["o"].b[2]}-'), '-3-')

    def test_space_is_not_ignored_after_newline_in_extend_variable_syntax(self):
        self.varz['${x}'] = 'test string'
        self.varz['${lf}'] = '\\n'
        self.varz['${lfs}'] = '\\n '
        for inp, exp in [('${x.replace(" ", """\\n""")}', 'test\nstring'),
                         ('${x.replace(" ", """\\n """)}', 'test\n string'),
                         ('${x.replace(" ", """${lf}""")}', 'test\nstring'),
                         ('${x.replace(" ", """${lfs}""")}', 'test\n string')]:
            assert_equals(self.varz.replace_scalar(inp), exp)

    def test_escaping_with_extended_variable_syntax(self):
        self.varz['${p}'] = 'c:\\temp'
        assert self.varz['${p}'] == 'c:\\temp'
        assert_equals(self.varz.replace_scalar('${p + "\\\\foo.txt"}'),
                      'c:\\temp\\foo.txt')

    def test_internal_variables(self):
        # Internal variables are variables like ${my${name}}
        self.varz['${name}'] = 'name'
        self.varz['${my name}'] = 'value'
        assert_equals(self.varz.replace_scalar('${my${name}}'), 'value')
        self.varz['${whos name}'] = 'my'
        assert_equals(self.varz.replace_scalar('${${whos name} ${name}}'), 'value')
        assert_equals(self.varz.replace_scalar('${${whos${name}}${name}}'), 'value')
        self.varz['${my name}'] = [1,2,3]
        assert_equals(self.varz.replace_scalar('${${whos${name}}${name}}'), [1,2,3])
        assert_equals(self.varz.replace_scalar('- ${${whos${name}}${name}} -'), '- [1, 2, 3] -')

    def test_math_with_internal_vars(self):
        assert_equals(self.varz.replace_scalar('${${1}+${2}}'), 3)
        assert_equals(self.varz.replace_scalar('${${1}-${2}}'), -1)
        assert_equals(self.varz.replace_scalar('${${1}*${2}}'), 2)
        assert_equals(self.varz.replace_scalar('${${1}/${2}}'), 0)

    def test_math_with_internal_vars_with_spaces(self):
        assert_equals(self.varz.replace_scalar('${${1} + ${2.5}}'), 3.5)
        assert_equals(self.varz.replace_scalar('${${1} - ${2} + 1}'), 0)
        assert_equals(self.varz.replace_scalar('${${1} * ${2} - 1}'), 1)
        assert_equals(self.varz.replace_scalar('${${1} / ${2.0}}'), 0.5)

    def test_math_with_internal_vars_does_not_work_if_first_var_is_float(self):
        assert_raises(DataError, self.varz.replace_scalar, '${${1.1}+${2}}')
        assert_raises(DataError, self.varz.replace_scalar, '${${1.1} - ${2}}')
        assert_raises(DataError, self.varz.replace_scalar, '${${1.1} * ${2}}')
        assert_raises(DataError, self.varz.replace_scalar, '${${1.1}/${2}}')

    def test_list_variable_as_scalar(self):
        self.varz['@{name}'] = exp = ['spam', 'eggs']
        assert_equals(self.varz.replace_scalar('${name}'), exp)
        assert_equals(self.varz.replace_list(['${name}', 42]), [exp, 42])
        assert_equals(self.varz.replace_string('${name}'), str(exp))
        assert_true(self.varz.has_key('${name}'))

    def test_copy(self):
        varz = variables.Variables(identifiers=['$'])
        varz['${foo}'] = 'bar'
        copy = varz.copy()
        assert_equals(copy['${foo}'], 'bar')
        assert_equals(copy._identifiers, ['$'])

    if utils.is_jython:

        def test_variable_as_object_in_java(self):
            obj = JavaObject('hello')
            self.varz['${obj}'] = obj
            assert_equals(self.varz['${obj}'], obj)
            assert_equals(self.varz.replace_scalar('${obj} world'), 'hello world')

        def test_extended_variables_in_java(self):
            obj = JavaObject('my name')
            self.varz['${obj}'] = obj
            assert_equals(self.varz.replace_list(['${obj.name}']), ['my name'])

class TestVariableSplitter(unittest.TestCase):

    _identifiers = ['$','@','%','&','*']

    def test_empty(self):
        self._test('', None)

    def test_no_vars(self):
        for inp in ['hello world', '$hello', '{hello}', '$\\{hello}',
                    '${hello', '$hello}' ]:
            self._test(inp, None)

    def test_backslashes(self):
        for inp in ['\\', '\\\\', '\\\\\\\\\\',
                    '\\hello\\\\world\\\\\\']:
            self._test(inp, None)

    def test_one_var(self):
        self._test('${hello}', '${hello}', 0)
        self._test('1 @{hello} more', '@{hello}', 2)
        self._test('*{hi}}', '*{hi}', 0)
        self._test('{%{{hi}}', '%{{hi}', 1)
        self._test('-= ${} =-', '${}', 3)
        # In this case splitter thinks there are internal but there aren't.
        # Better check would probably spent more time than that is saved when
        # variable base is processed again in this special case.
        self._test('%{hi%{u}', '%{hi%{u}', 0, internal=True)

    def test_multiple_vars(self):
        self._test('${hello} ${world}', '${hello}', 0)
        self._test('hi %{u}2 and @{u2} and also *{us3}', '%{u}', 3)
        self._test('0123456789 %{1} and @{2', '%{1}', 11)

    def test_escaped_var(self):
        self._test('\\${hello}', None)
        self._test('hi \\\\\\${hello} moi', None)

    def test_not_escaped_var(self):
        self._test('\\\\${hello}', '${hello}', 2)
        self._test('\\hi \\\\\\\\\\\\${hello} moi', '${hello}',
                   len('\\hi \\\\\\\\\\\\'))
        self._test('\\ ${hello}', '${hello}', 2)
        self._test('${hello}\\', '${hello}', 0)
        self._test('\\ \\ ${hel\\lo}\\', '${hel\\lo}', 4)

    def test_escaped_and_not_escaped_vars(self):
        for inp, var, start in [
                ('\\${esc} ${not}', '${not}', len('\\${esc} ')),
                ('\\\\\\${esc} \\\\${not}', '${not}',
                 len('\\\\\\${esc} \\\\')),
                ('\\${esc}\\\\${not}${n2}', '${not}', len('\\${esc}\\\\')) ]:
            self._test(inp, var, start)

    def test_internal_vars(self):
        for inp, var, start in [
                ('${hello${hi}}', '${hello${hi}}', 0),
                ('bef ${${hi}hello} aft', '${${hi}hello}', 4),
                ('\\${not} ${hel${hi}lo} ', '${hel${hi}lo}', len('\\${not} ')),
                ('${${hi}${hi}}\\', '${${hi}${hi}}', 0),
                ('${${hi${hi}}} ${xx}', '${${hi${hi}}}', 0),
                ('${xx} ${${hi${hi}}}', '${xx}', 0),
                ('${\\${hi${hi}}}', '${\\${hi${hi}}}', 0),
                ('\\${${hi${hi}}}', '${hi${hi}}', len('\\${')),
                ('\\${\\${hi\\\\${hi}}}', '${hi}', len('\\${\\${hi\\\\')) ]:
            internal = var.count('{') > 1
            self._test(inp, var, start, internal=internal)

    def test_index(self):
        self._test('@{x}[0]', '@{x}', 0, '0')
        self._test('.@{x}[42]..', '@{x}', 1, '42')
        self._test('@{x}[${i}] ${xyz}', '@{x}', 0, '${i}')
        self._test('@{x}[]', '@{x}', 0, '')
        self._test('@{x}[inv]', '@{x}', 0, 'inv')
        self._test('@{x}[0', '@{x}', 0, None)
        self._test('@{x}}[0]', '@{x}', 0, None)
        self._test('${x}[0]', '${x}', 0, None)
        self._test('%{x}[0]', '%{x}', 0, None)
        self._test('*{x}[0]', '*{x}', 0, None)
        self._test('&{x}[0]', '&{x}', 0, None)

    def test_custom_identifiers(self):
        for inp, start in [ ('@{x}${y}', 4),
                            ('%{x} ${y}', 5),
                            ('*{x}567890${y}', 10),
                            ('&{x}%{x}@{x}\\${x}${y}',
                             len('&{x}%{x}@{x}\\${x}')) ]:
            self._test(inp, '${y}', start, identifiers=['$'])

    def test_identifier_as_variable_name(self):
        for i in self._identifiers:
            for count in 1,2,3,42:
                var = '%s{%s}' % (i, i*count)
                self._test(var, var)
                self._test(var+'spam', var)
                self._test('eggs'+var+'spam', var, start=4)
                self._test(i+var+i, var, start=1)

    def test_identifier_as_variable_name_with_internal_vars(self):
        for i in self._identifiers:
            for count in 1,2,3,42:
                var = '%s{%s{%s}}' % (i, i*count, i)
                self._test(var, var, internal=True)
                self._test('eggs'+var+'spam', var, start=4, internal=True)
                var = '%s{%s{%s}}' % (i, i*count, i*count)
                self._test(var, var, internal=True)
                self._test('eggs'+var+'spam', var, start=4, internal=True)

    def _test(self, inp, variable, start=0, index=None, identifiers=None,
              internal=False):
        if variable is not None:
            identifier = variable[0]
            base = variable[2:-1]
            end = start + len(variable)
            if index is not None:
                end += len(index) + 2
        else:
            identifier = base = None
            start = end = -1
        if not identifiers:
            identifiers = self._identifiers
        res = variables.VariableSplitter(inp, identifiers)
        assert_equals(res.base, base, "'%s' base" % inp)
        assert_equals(res.start, start, "'%s' start" % inp)
        assert_equals(res.end, end, "'%s' end" % inp)
        assert_equals(res.identifier, identifier, "'%s' indentifier" % inp)
        assert_equals(res.index, index, "'%s' index" % inp)
        assert_equals(res._may_have_internal_variables, internal,
                      "'%s' internal" % inp)

if __name__ == '__main__':
    unittest.main()