#! /usr/bin/python3
# -*- coding: utf-8 -*-

# https://docs.docker.com/compose/compose-file/#service-configuration-reference
# https://docs.docker.com/samples/
# https://docs.docker.com/compose/gettingstarted/
# https://docs.docker.com/compose/django/
# https://docs.docker.com/compose/wordpress/

from __future__ import print_function

import sys
import os
import argparse
import subprocess
import time
import re
import hashlib
import random

from threading import Thread

try:
    from shlex import quote as cmd_quote
except ImportError:
    from pipes import quote as cmd_quote

# import fnmatch
# fnmatch.fnmatchcase(env, "*_HOST")

import json
import yaml

__version__ = '0.1.5'

PY3 = sys.version_info[0] == 3
if PY3:
    basestring = str

# helper functions

is_str  = lambda s: isinstance(s, basestring)
is_dict = lambda d: isinstance(d, dict)
is_list = lambda l: not is_str(l) and not is_dict(l) and hasattr(l, "__iter__")
# identity filter
filteri = lambda a: filter(lambda i:i, a)

def try_int(i, fallback=None):
    try:
        return int(i)
    except ValueError:
        pass
    except TypeError:
        pass
    return fallback

dir_re = re.compile("^[~/\.]")
propagation_re=re.compile("^(?:z|Z|r?shared|r?slave|r?private)$")

# NOTE: if a named volume is used but not defined it gives
# ERROR: Named volume "so and so" is used in service "xyz" but no declaration was found in the volumes section.
# unless it's anon-volume

def parse_short_mount(mount_str, basedir):
    mount_a = mount_str.split(':')
    mount_opt_dict = {}
    mount_opt = None
    if len(mount_a)==1:
        # Just specify a path and let the Engine create a volume
        # - /var/lib/mysql
        mount_src, mount_dst=None, mount_str
    elif len(mount_a)==2:
        mount_src, mount_dst = mount_a
        # dest must start with /, otherwise it's option
        if not mount_dst.startswith('/'):
            mount_dst, mount_opt = mount_a
            mount_src = None
    elif len(mount_a)==3:
        mount_src, mount_dst, mount_opt = mount_a
    else:
        raise ValueError("could not parse mount "+mount_str)
    if mount_src and dir_re.match(mount_src):
        # Specify an absolute path mapping
        # - /opt/data:/var/lib/mysql
        # Path on the host, relative to the Compose file
        # - ./cache:/tmp/cache
        # User-relative path
        # - ~/configs:/etc/configs/:ro
        mount_type = "bind"
        # TODO: should we use os.path.realpath(basedir)?
        mount_src = os.path.join(basedir, os.path.expanduser(mount_src))
    else:
        # Named volume
        # - datavolume:/var/lib/mysql
        mount_type = "volume"
    mount_opts = filteri((mount_opt or '').split(','))
    for opt in mount_opts:
        if opt=='ro': mount_opt_dict["read_only"]=True
        elif opt=='rw': mount_opt_dict["read_only"]=False
        elif propagation_re.match(opt): mount_opt_dict["bind"]=dict(propagation=opt)
        else:
            # TODO: ignore
            raise ValueError("unknown mount option "+opt)
    return dict(type=mount_type, source=mount_src, target=mount_dst, **mount_opt_dict)

def fix_mount_dict(mount_dict, proj_name, srv_name):
    """
    in-place fix mount dictionary to:
    - add missing source
    - prefix source with proj_name
    """
    if mount_dict["type"]=="volume":
        source = mount_dict.get("source")
        # keep old source
        mount_dict["_source"] = source
        if not source:
            # missing source
            mount_dict["source"] = "_".join([
                proj_name, srv_name,
                hashlib.md5(mount_dict["target"].encode("utf-8")).hexdigest(),
            ])
        else:
            # prefix with proj_name
            mount_dict["source"] = proj_name+"_"+source
    return mount_dict

# docker and docker-compose support subset of bash variable substitution
# https://docs.docker.com/compose/compose-file/#variable-substitution
# https://docs.docker.com/compose/env-file/
# https://www.gnu.org/software/bash/manual/html_node/Shell-Parameter-Expansion.html
# $VARIABLE
# ${VARIABLE}
# ${VARIABLE:-default} default if not set or empty
# ${VARIABLE-default} default if not set
# ${VARIABLE:?err} raise error if not set or empty
# ${VARIABLE?err} raise error if not set
# $$ means $

var_re = re.compile(r'\$(\{(?:[^\s\$:\-\}]+)\}|(?:[^\s\$\{\}]+))')
var_def_re = re.compile(r'\$\{([^\s\$:\-\}]+)(:)?-([^\}]+)\}')
var_err_re = re.compile(r'\$\{([^\s\$:\-\}]+)(:)?\?([^\}]+)\}')

def dicts_get(dicts, key, fallback='', fallback_empty=False):
    """
    get the given key from any dict in dicts, trying them one by one
    if not found in any, then use fallback, if fallback is Exception raise is
    """
    value = None
    for d in dicts:
        value = d.get(key)
        if value is not None: break
    if not value:
        if fallback_empty or value is None:
            value = fallback
        if isinstance(value, Exception):
            raise value
    return value

def rec_subs(value, dicts):
    """
    do bash-like substitution in value and if list of dictionary do that recursively
    """
    if is_dict(value):
        value = dict([(k, rec_subs(v, dicts)) for k, v in value.items()])
    elif is_str(value):
        value = var_re.sub(lambda m: dicts_get(dicts, m.group(1).strip('{}')), value)
        sub_def = lambda m: dicts_get(dicts, m.group(1), m.group(3), m.group(2) == ':')
        value = var_def_re.sub(sub_def, value)
        sub_err = lambda m: dicts_get(dicts, m.group(1), RuntimeError(m.group(3)),
                                      m.group(2) == ':')
        value = var_err_re.sub(sub_err, value)
        value = value.replace('$$', '$')
    elif hasattr(value, "__iter__"):
        value = [rec_subs(i, dicts) for i in value]
    return value

def norm_as_list(src):
    """
    given a dictionary {key1:value1, key2: None} or list
    return a list of ["key1=value1", "key2"]
    """
    if src is None:
        dst = []
    elif is_dict(src):
        dst = [("{}={}".format(k, v) if v else k) for k, v in src.items()]
    elif is_list(src):
        dst = list(src)
    else:
        dst = [src]
    return dst

def norm_as_dict(src):
    """
    given a list ["key1=value1", "key2"]
    return a dictionary {key1:value1, key2: None}
    """
    if src is None:
        dst = {}
    elif is_dict(src):
        dst = dict(src)
    elif is_list(src):
        dst = [i.split("=", 1) for i in src if i]
        dst = dict([(a if len(a) == 2 else (a[0], None)) for a in dst])
    elif is_str(src):
        key, value = src.split("=", 1) if "=" in src else (src, None)
        dst = {key: value}
    else:
        raise ValueError("dictionary or iterable is expected")
    return dst

def norm_ulimit(inner_value):
    if is_dict(inner_value):
        if not inner_value.keys() & {"soft", "hard"}:
            raise ValueError("expected at least one soft or hard limit")
        soft = inner_value.get("soft", inner_value.get("hard"))
        hard = inner_value.get("hard", inner_value.get("soft"))
        return "{}:{}".format(soft, hard)
    elif is_list(inner_value): return norm_ulimit(norm_as_dict(inner_value))
    # if int or string return as is
    return inner_value

# transformation helpers

def adj_hosts(services, cnt, dst="127.0.0.1"):
    """
    adjust container cnt in-place to add hosts pointing to dst for services
    """
    common_extra_hosts = []
    for srv, cnts in services.items():
        common_extra_hosts.append("{}:{}".format(srv, dst))
        for cnt0 in cnts:
            common_extra_hosts.append("{}:{}".format(cnt0, dst))
    extra_hosts = list(cnt.get("extra_hosts", []))
    extra_hosts.extend(common_extra_hosts)
    # link aliases
    for link in cnt.get("links", []):
        a = link.strip().split(':', 1)
        if len(a) == 2:
            alias = a[1].strip()
            extra_hosts.append("{}:{}".format(alias, dst))
    cnt["extra_hosts"] = extra_hosts

def move_list(dst, containers, key):
    """
    move key (like port forwarding) from containers to dst (a pod or a infra container)
    """
    a = set(dst.get(key) or [])
    for cnt in containers:
        a0 = cnt.get(key)
        if a0:
            a.update(a0)
            del cnt[key]
    if a:
        dst[key] = list(a)

def move_port_fw(dst, containers):
    """
    move port forwarding from containers to dst (a pod or a infra container)
    """
    move_list(dst, containers, "ports")

def move_extra_hosts(dst, containers):
    """
    move port forwarding from containers to dst (a pod or a infra container)
    """
    move_list(dst, containers, "extra_hosts")

# transformations

transformations = {}

def trans(func):
    transformations[func.__name__.replace("tr_", "")] = func
    return func

@trans
def tr_identity(project_name, services, given_containers):
    containers = []
    for cnt in given_containers:
        containers.append(dict(cnt))
    return [], containers

@trans
def tr_publishall(project_name, services, given_containers):
    containers = []
    for cnt0 in given_containers:
        cnt = dict(cnt0, publishall=True)
        # adjust hosts to point to the gateway, TODO: adjust host env
        adj_hosts(services, cnt, '10.0.2.2')
        containers.append(cnt)
    return [], containers

@trans
def tr_hostnet(project_name, services, given_containers):
    containers = []
    for cnt0 in given_containers:
        cnt = dict(cnt0, network_mode="host")
        # adjust hosts to point to localhost, TODO: adjust host env
        adj_hosts(services, cnt, '127.0.0.1')
        containers.append(cnt)
    return [], containers

@trans
def tr_cntnet(project_name, services, given_containers):
    containers = []
    infra_name = project_name + "_infra"
    infra = dict(
        name=infra_name,
        image="k8s.gcr.io/pause:3.1",
    )
    for cnt0 in given_containers:
        cnt = dict(cnt0, network_mode="container:"+infra_name)
        deps = cnt.get("depends_on") or []
        deps.append(infra_name)
        cnt["depends_on"] = deps
        # adjust hosts to point to localhost, TODO: adjust host env
        adj_hosts(services, cnt, '127.0.0.1')
        if "hostname" in cnt:
            del cnt["hostname"]
        containers.append(cnt)
    move_port_fw(infra, containers)
    move_extra_hosts(infra, containers)
    containers.insert(0, infra)
    return [], containers

@trans
def tr_1pod(project_name, services, given_containers):
    """
    project_name:
    services: {service_name: ["container_name1", "..."]}, currently only one is supported
    given_containers: [{}, ...]
    """
    pod = dict(name=project_name)
    containers = []
    for cnt0 in given_containers:
        cnt = dict(cnt0, pod=project_name)
        # services can be accessed as localhost because they are on one pod
        # adjust hosts to point to localhost, TODO: adjust host env
        adj_hosts(services, cnt, '127.0.0.1')
        containers.append(cnt)
    return [pod], containers

@trans
def tr_1podfw(project_name, services, given_containers):
    pods, containers = tr_1pod(project_name, services, given_containers)
    pod = pods[0]
    move_port_fw(pod, containers)
    return pods, containers

def mount_dict_vol_to_bind(compose, mount_dict):
    """
    inspect volume to get directory
    create volume if needed
    and return mount_dict as bind of that directory
    """
    proj_name = compose.project_name
    shared_vols = compose.shared_vols
    if mount_dict["type"]!="volume": return mount_dict
    vol_name_orig = mount_dict.get("_source", None)
    vol_name = mount_dict["source"]
    print("podman volume inspect {vol_name} || podman volume create {vol_name}".format(vol_name=vol_name))
    # podman volume list --format '{{.Name}}\t{{.MountPoint}}' -f 'label=io.podman.compose.project=HERE'
    try: out = compose.podman.output(["volume", "inspect", vol_name]).decode('utf-8')
    except subprocess.CalledProcessError:
        compose.podman.output(["volume", "create", "--label", "io.podman.compose.project={}".format(proj_name), vol_name])
        out = compose.podman.output(["volume", "inspect", vol_name]).decode('utf-8')
    src = json.loads(out)[0]["mountPoint"]
    ret=dict(mount_dict, type="bind", source=src, _vol=vol_name)
    bind_prop=ret.get("bind", {}).get("propagation")
    if not bind_prop:
        if "bind" not in ret:
            ret["bind"]={}
        # if in top level volumes then it's shared bind-propagation=z
        if vol_name_orig and vol_name_orig in shared_vols:
            ret["bind"]["propagation"]="z"
        else:
            ret["bind"]["propagation"]="Z"
    try: del ret["volume"]
    except KeyError: pass
    return ret

def mount_desc_to_args(compose, mount_desc, srv_name, cnt_name):
    basedir = compose.dirname
    proj_name = compose.project_name
    shared_vols = compose.shared_vols
    if is_str(mount_desc): mount_desc=parse_short_mount(mount_desc, basedir)
    # not needed
    # podman support: podman run --rm -ti --mount type=volume,source=myvol,destination=/delme busybox
    mount_desc = mount_dict_vol_to_bind(compose, fix_mount_dict(mount_desc, proj_name, srv_name))
    mount_type = mount_desc.get("type")
    source = mount_desc.get("source")
    target = mount_desc["target"]
    opts=[]
    if mount_desc.get("bind"):
        bind_prop=mount_desc["bind"].get("propagation")
        if bind_prop: opts.append("bind-propagation={}".format(bind_prop))
    if mount_desc.get("read_only", False): opts.append("ro")
    if mount_type=='tmpfs':
        tmpfs_opts = mount_desc.get("tmpfs", {})
        tmpfs_size = tmpfs_opts.get("size")
        if tmpfs_size:
            opts.append("tmpfs-size={}".format(tmpfs_size))
        tmpfs_mode = tmpfs_opts.get("mode")
        if tmpfs_mode:
            opts.append("tmpfs-mode={}".format(tmpfs_mode))
    opts=",".join(opts)
    if mount_type=='bind':
        return "type=bind,source={source},destination={target},{opts}".format(
            source=source,
            target=target,
            opts=opts
        ).rstrip(",")
    elif mount_type=='tmpfs':
        return "type=tmpfs,destination={target},{opts}".format(
            target=target,
            opts=opts
        ).rstrip(",")
    else:
        raise ValueError("unknown mount type:"+mount_type)

def container_to_args(compose, cnt, detached=True, podman_command='run'):
    # TODO: double check -e , --add-host, -v, --read-only
    dirname = compose.dirname
    shared_vols = compose.shared_vols
    pod = cnt.get('pod') or ''
    podman_args = [
        podman_command,
        '--name={}'.format(cnt.get('name')),
    ]

    if detached:
        podman_args.append("-d")

    if pod:
        podman_args.append('--pod={}'.format(pod))
    sec = norm_as_list(cnt.get("security_opt"))
    for s in sec:
        podman_args.extend(['--security-opt', s])
    if cnt.get('read_only'):
        podman_args.append('--read-only')
    for i in cnt.get('labels', []):
        podman_args.extend(['--label', i])
    net = cnt.get("network_mode")
    if net:
        podman_args.extend(['--network', net])
    env = norm_as_list(cnt.get('environment', {}))
    for d in cnt.get('devices', []):
        podman_args.extend(['--device', d])
    for e in env:
        podman_args.extend(['-e', e])
    for i in cnt.get('env_file', []):
        i = os.path.realpath(os.path.join(dirname, i))
        podman_args.extend(['--env-file', i])
    tmpfs_ls = cnt.get('tmpfs', [])
    if is_str(tmpfs_ls): tmpfs_ls=[tmpfs_ls]
    for i in tmpfs_ls:
        podman_args.extend(['--tmpfs', i])
    for volume in cnt.get('volumes', []):
        # TODO: should we make it os.path.realpath(os.path.join(, i))?
        mount_args = mount_desc_to_args(compose, volume, cnt['_service'], cnt['name'])
        podman_args.extend(['--mount', mount_args])
    for i in cnt.get('extra_hosts', []):
        podman_args.extend(['--add-host', i])
    for i in cnt.get('expose', []):
        podman_args.extend(['--expose', i])
    if cnt.get('publishall'):
        podman_args.append('-P')
    for i in cnt.get('ports', []):
        podman_args.extend(['-p', i])
    user = cnt.get('user')
    if user is not None:
        podman_args.extend(['-u', user])
    if cnt.get('working_dir') is not None:
        podman_args.extend(['-w', cnt.get('working_dir')])
    if cnt.get('hostname'):
        podman_args.extend(['--hostname', cnt.get('hostname')])
    if cnt.get('shm_size'):
        podman_args.extend(['--shm_size', '{}'.format(cnt.get('shm_size'))])
    if cnt.get('stdin_open'):
        podman_args.append('-i')
    if cnt.get('tty'):
        podman_args.append('--tty')
    ulimit = cnt.get('ulimit', [])
    if ulimit is not None:
        # ulimit can be a single value, i.e. ulimit: host
        if is_str(ulimit):
            podman_args.extend(['--ulimit', ulimit])
        # or a dictionary or list:
        else:
            ulimit = norm_as_dict(ulimit)
            ulimit = [ "{}={}".format(ulimit_key, norm_ulimit(inner_value)) for ulimit_key, inner_value in ulimit.items()]
            for i in ulimit:
                podman_args.extend(['--ulimit', i])
    # currently podman shipped by fedora does not package this
    # if cnt.get('init'):
    #    args.append('--init')
    entrypoint = cnt.get('entrypoint')
    if entrypoint is not None:
        if is_str(entrypoint):
            podman_args.extend(['--entrypoint', entrypoint])
        else:
            podman_args.extend(['--entrypoint', json.dumps(entrypoint)])

    # WIP: healthchecks are still work in progress
    healthcheck = cnt.get('healthcheck', None) or {}
    if not is_dict(healthcheck):
        raise ValueError("'healthcheck' must be an key-value mapping")
    healthcheck_test = healthcheck.get('test')
    if healthcheck_test:
        # If it's a string, it's equivalent to specifying CMD-SHELL
        if is_str(healthcheck_test):
            # podman does not add shell to handle command with whitespace
            podman_args.extend(['--healthcheck-command', '/bin/sh -c {}'.format(cmd_quote(healthcheck_test))])
        elif is_list(healthcheck_test):
            # If it's a list, first item is either NONE, CMD or CMD-SHELL.
            healthcheck_type = healthcheck_test.pop(0)
            if healthcheck_type == 'NONE':
                podman_args.append("--no-healthcheck")
            elif healthcheck_type == 'CMD':
                podman_args.extend(['--healthcheck-command', '/bin/sh -c {}'.format(
                    "' '".join([cmd_quote(i) for i in healthcheck_test])
                )])
            elif healthcheck_type == 'CMD-SHELL':
                if len(healthcheck_test)!=1:
                    raise ValueError("'CMD_SHELL' takes a single string after it")
                podman_args.extend(['--healthcheck-command', '/bin/sh -c {}'.format(cmd_quote(healthcheck_test[0]))])
            else:
                raise ValueError(
                    "unknown healthcheck test type [{}],\
                     expecting NONE, CMD or CMD-SHELL."
                     .format(healthcheck_type)
                )
        else:
            raise ValueError("'healthcheck.test' either a string or a list")

    # interval, timeout and start_period are specified as durations.
    if 'interval' in healthcheck:
        podman_args.extend(['--healthcheck-interval', healthcheck['interval']])
    if 'timeout' in healthcheck:
        podman_args.extend(['--healthcheck-timeout', healthcheck['timeout']])
    if 'start_period' in healthcheck:
        podman_args.extend(['--healthcheck-start-period', healthcheck['start_period']])

    # convert other parameters to string
    if 'retries' in healthcheck:
        podman_args.extend(['--healthcheck-retries', '{}'.format(healthcheck['retries'])])

    podman_args.append(cnt.get('image'))  # command, ..etc.
    command = cnt.get('command')
    if command is not None:
        if is_str(command):
            podman_args.extend(command.split())
        else:
            podman_args.extend(command)
    return podman_args

def rec_deps(services, container_by_name, cnt, init_service):
    deps = cnt["_deps"]
    for dep in deps.copy():
        dep_cnts = services.get(dep)
        if not dep_cnts:
            continue
        dep_cnt = container_by_name.get(dep_cnts[0])
        if dep_cnt:
            # TODO: avoid creating loops, A->B->A
            if init_service and init_service in dep_cnt["_deps"]:
                continue
            new_deps = rec_deps(services, container_by_name,
                                dep_cnt, init_service)
            deps.update(new_deps)
    return deps

def flat_deps(services, container_by_name):
    for name, cnt in container_by_name.items():
        deps = set([(c.split(":")[0] if ":" in c else c)
                    for c in cnt.get("links", [])])
        deps.update(cnt.get("depends_on", []))
        cnt["_deps"] = deps
    for name, cnt in container_by_name.items():
        rec_deps(services, container_by_name, cnt, cnt.get('_service'))

###################
# podman and compose classes
###################

class Podman:
    def __init__(self, compose, podman_path='podman', dry_run=False):
        self.compose = compose
        self.podman_path = podman_path
        self.dry_run = dry_run

    def output(self, podman_args):
        cmd = [self.podman_path]+podman_args
        return subprocess.check_output(cmd)

    def run(self, podman_args, wait=True, sleep=1):
        podman_args_str = [str(arg) for arg in podman_args]
        print("podman " + " ".join(podman_args_str))
        if self.dry_run:
            return None
        cmd = [self.podman_path]+podman_args_str
        # subprocess.Popen(args, bufsize = 0, executable = None, stdin = None, stdout = None, stderr = None, preexec_fn = None, close_fds = False, shell = False, cwd = None, env = None, universal_newlines = False, startupinfo = None, creationflags = 0)
        p = subprocess.Popen(cmd)
        if wait:
            print(p.wait())
        if sleep:
            time.sleep(sleep)
        return p

def normalize(compose):
    """
    convert compose dict of some keys from string or dicts into arrays
    """
    services = compose.get("services", None) or {}
    for service_name, service in services.items():
        for key in ("env_file", "security_opt"):
            if key not in service: continue
            if is_str(service[key]): service[key]=[service[key]]
        for key in ("environment", "labels"):
            if key not in service: continue
            service[key] = norm_as_dict(service[key])
    return compose

def rec_merge(target, source):
    """
    update content of compose with keys from content recursively
    """
    done = set()
    for key, value in source.items():
        if key in target: continue
        target[key]=value
        done.add(key)
    for key, value in target.items():
        if key in done: continue
        if key not in source: continue
        value2 = source[key]
        if type(value2)!=type(value):
            raise ValueError("can't merge value of {} of type {} and {}".format(key, type(value), type(value2)))
        if is_str(value2):
            target[key]=value2
        elif is_list(value2):
            value.extend(value2)
        elif is_dict(value2):
            rec_merge(value, value2)
        else:
            raise ValueError("unexpected type of {}".format(key))
    return target

class PodmanCompose:
    def __init__(self):
        self.commands = {}
        self.global_args = None
        self.project_name = None
        self.dirname = None
        self.pods = None
        self.containers = None
        self.shared_vols = None
        self.container_names_by_service = None
        self.container_by_name = None

    def run(self):
        args = self._parse_args()
        self._parse_compose_file()
        podman_path = args.podman_path
        if podman_path != 'podman':
            if os.path.isfile(podman_path) and os.access(podman_path, os.X_OK):
                podman_path = os.path.realpath(podman_path)
            else:
                # this also works if podman hasn't been installed now
                if dry_run == False:
                    raise IOError(
                        "Binary {} has not been found.".format(podman_path))

        self.podman = Podman(self, podman_path, args.dry_run)
        cmd_name = args.command
        cmd = self.commands[cmd_name]
        cmd(self, args)

    def _parse_compose_file(self):
        args = self.global_args
        cmd = args.command
        if not args.file:
            args.file = list(filter(os.path.exists, [
                "docker-compose.yml",
                "docker-compose.yaml",
                "docker-compose.override.yml",
                "docker-compose.override.yaml",
                "container-compose.yml",
                "container-compose.yaml",
                "container-compose.override.yml",
                "container-compose.override.yaml"
            ]))
        files = args.file
        if not files:
            print("no docker-compose.yml or container-compose.yml file found, pass files with -f")
        ex = map(os.path.exists, files)
        missing = [ fn0 for ex0, fn0 in zip(ex, files) if not ex0 ]
        if missing:
            print("missing files: ", missing)
            exit(1)
        # make absolute
        files = list(map(os.path.realpath, files))
        filename = files[0]
        project_name = args.project_name
        no_ansi = args.no_ansi
        no_cleanup = args.no_cleanup
        dry_run = args.dry_run
        transform_policy = args.transform_policy
        host_env = None
        dirname = os.path.dirname(filename)
        dir_basename = os.path.basename(dirname)
        self.dirname = dirname
        # TODO: remove next line
        os.chdir(dirname)

        if not project_name:
            project_name = dir_basename
        self.project_name = project_name

        dotenv_path = os.path.join(dirname, ".env")
        if os.path.exists(dotenv_path):
            with open(dotenv_path, 'r') as f:
                dotenv_ls = [l.strip() for l in f if l.strip() and not l.startswith('#')]
                dotenv_dict = dict([l.split("=", 1) for l in dotenv_ls if "=" in l])
        else:
            dotenv_dict = {}
        compose={'_dirname': dirname}
        for filename in files:
            with open(filename, 'r') as f:
                content = yaml.safe_load(f)
                #print(filename, json.dumps(content, indent = 2))
                content = normalize(content)
                #print(filename, json.dumps(content, indent = 2))
                content = rec_subs(content, [os.environ, dotenv_dict])
                rec_merge(compose, content)
        # debug mode
        if len(files)>1:
            print(" ** merged:\n", json.dumps(compose, indent = 2))
        ver = compose.get('version')
        services = compose.get('services')
        # volumes: [...]
        shared_vols = compose.get('volumes', {})
        # shared_vols = list(shared_vols.keys())
        shared_vols = set(shared_vols.keys())
        self.shared_vols = shared_vols
        podman_compose_labels = [
            "io.podman.compose.config-hash=123",
            "io.podman.compose.project=" + project_name,
            "io.podman.compose.version=0.0.1",
        ]
        # other top-levels:
        # networks: {driver: ...}
        # configs: {...}
        # secrets: {...}
        given_containers = []
        container_names_by_service = {}
        for service_name, service_desc in services.items():
            replicas = try_int(service_desc.get('deploy', {}).get('replicas', '1'))
            container_names_by_service[service_name] = []
            for num in range(1, replicas+1):
                name0 = "{project_name}_{service_name}_{num}".format(
                    project_name=project_name,
                    service_name=service_name,
                    num=num,
                )
                if num == 1:
                    name = service_desc.get("container_name", name0)
                else:
                    name = name0
                container_names_by_service[service_name].append(name)
                # print(service_name,service_desc)
                cnt = dict(name=name, num=num,
                           service_name=service_name, **service_desc)
                if 'image' not in cnt:
                    cnt['image'] = "{project_name}_{service_name}".format(
                        project_name=project_name,
                        service_name=service_name,
                    )
                labels = norm_as_list(cnt.get('labels'))
                labels.extend(podman_compose_labels)
                labels.extend([
                    "com.docker.compose.container-number={}".format(num),
                    "com.docker.compose.service=" + service_name,
                ])
                cnt['labels'] = labels
                cnt['_service'] = service_name
                cnt['_project'] = project_name
                given_containers.append(cnt)
        self.container_names_by_service = container_names_by_service
        container_by_name = dict([(c["name"], c) for c in given_containers])
        flat_deps(container_names_by_service, container_by_name)
        #print("deps:", [(c["name"], c["_deps"]) for c in given_containers])
        given_containers = list(container_by_name.values())
        given_containers.sort(key=lambda c: len(c.get('_deps') or []))
        #print("sorted:", [c["name"] for c in given_containers])
        tr = transformations[transform_policy]
        pods, containers = tr(
            project_name, container_names_by_service, given_containers)
        self.pods = pods
        self.containers = containers
        self.container_by_name = dict([ (c["name"], c) for c in containers])

    def _parse_args(self):
        parser = argparse.ArgumentParser()
        self._init_global_parser(parser)
        subparsers = parser.add_subparsers(title='command', dest='command')
        subparser = subparsers.add_parser('help', help='show help')
        for cmd_name, cmd in self.commands.items():
            subparser = subparsers.add_parser(cmd_name, help=cmd._cmd_desc)
            for cmd_parser in cmd._parse_args:
                cmd_parser(subparser)
        self.global_args = parser.parse_args()
        if not self.global_args.command or self.global_args.command=='help':
            parser.print_help()
            exit(-1)
        return self.global_args

    def _init_global_parser(self, parser):
        parser.add_argument("-f", "--file",
                            help="Specify an alternate compose file (default: docker-compose.yml)",
                            metavar='file', action='append', default=[])
        parser.add_argument("-p", "--project-name",
                            help="Specify an alternate project name (default: directory name)",
                            type=str, default=None)
        parser.add_argument("--podman-path",
                            help="Specify an alternate path to podman (default: use location in $PATH variable)",
                            type=str, default="podman")
        parser.add_argument("--no-ansi",
                            help="Do not print ANSI control characters", action='store_true')
        parser.add_argument("--no-cleanup",
                            help="Do not stop and remove existing pod & containers", action='store_true')
        parser.add_argument("--dry-run",
                            help="No action; perform a simulation of commands", action='store_true')
        parser.add_argument("-t", "--transform_policy",
                            help="how to translate docker compose to podman [1pod|hostnet|accurate]",
                            choices=['1pod', '1podfw', 'hostnet', 'cntnet', 'publishall', 'identity'], default='1podfw')

podman_compose = PodmanCompose()

###################
# decorators to add commands and parse options
###################

class cmd_run:
    def __init__(self, compose, cmd_name, cmd_desc):
        self.compose = compose
        self.cmd_name = cmd_name
        self.cmd_desc = cmd_desc
    def __call__(self, func):
        def wrapped(*args, **kw):
            return func(*args, **kw)
        wrapped._compose = self.compose
        wrapped._cmd_name = self.cmd_name
        wrapped._cmd_desc = self.cmd_desc
        wrapped._parse_args = []
        self.compose.commands[self.cmd_name] = wrapped
        return wrapped

class cmd_parse:
    def __init__(self, compose, cmd_names):
        self.compose = compose
        self.cmd_names = cmd_names if is_list(cmd_names) else [cmd_names]

    def __call__(self, func):
        def wrapped(*args, **kw):
            return func(*args, **kw)
        for cmd_name in self.cmd_names:
            self.compose.commands[cmd_name]._parse_args.append(wrapped)
        return wrapped

###################
# actual commands
###################

@cmd_run(podman_compose, 'pull', 'pull stack images')
def compose_pull(compose, args):
    for cnt in compose.containers:
        if cnt.get('build'): continue
        compose.podman.run(["pull", cnt["image"]], sleep=0)

@cmd_run(podman_compose, 'push', 'push stack images')
def compose_push(compose, args):
    services = set(args.services)
    for cnt in compose.containers:
        if 'build' not in cnt: continue
        if services and cnt['_service'] not in services: continue
        compose.podman.run(["push", cnt["image"]], sleep=0)

def build_one(compose, args, cnt):
    if 'build' not in cnt: return
    if getattr(args, 'if_not_exists', None):
        try: img_id = compose.podman.output(['inspect', '-t', 'image', '-f', '{{.Id}}', cnt["image"]])
        except subprocess.CalledProcessError: img_id = None
        if img_id: return
    build_desc = cnt['build']
    if not hasattr(build_desc, 'items'):
        build_desc = dict(context=build_desc)
    ctx = build_desc.get('context', '.')
    dockerfile = os.path.join(ctx, build_desc.get("dockerfile", "Dockerfile"))
    if not os.path.exists(dockerfile):
        dockerfile = os.path.join(ctx, build_desc.get("dockerfile", "dockerfile"))
        if not os.path.exists(dockerfile):
            raise OSError("Dockerfile not found in "+ctx)
    build_args = [
        "build", "-t", cnt["image"],
        "-f", dockerfile
    ]
    if getattr(args, 'pull_always', None): build_args.append("--pull-always")
    elif getattr(args, 'pull', None): build_args.append("--pull")
    args_list = norm_as_list(build_desc.get('args', {}))
    for build_arg in args_list:
        build_args.extend(("--build-arg", build_arg,))
    build_args.append(ctx)
    compose.podman.run(build_args, sleep=0)

@cmd_run(podman_compose, 'build', 'build stack images')
def compose_build(compose, args):
    for cnt in compose.containers:
        build_one(compose, args, cnt)

def create_pods(compose, args):
    for pod in compose.pods:
        podman_args = [
            "pod", "create",
            "--name={}".format(pod["name"]),
            "--share", "net",
        ]
        ports = pod.get("ports") or []
        for i in ports:
            podman_args.extend(['-p', i])
        compose.podman.run(podman_args)

def up_specific(compose, args):
    deps = []
    if not args.no_deps:
        for service in args.services:
            deps.extend([])
    # args.always_recreate_deps
    print("services", args.services)
    raise NotImplementedError("starting specific services is not yet implemented")

@cmd_run(podman_compose, 'up', 'Create and start the entire stack or some of its services')
def compose_up(compose, args):
    if args.services:
        return up_specific(compose, args)

    if not args.no_build:
        # `podman build` does not cache, so don't always build
        build_args = argparse.Namespace(
            if_not_exists=(not args.build),
            **args.__dict__,
        )
        compose.commands['build'](compose, build_args)

    shared_vols = compose.shared_vols

    # TODO: implement check hash label for change
    if args.force_recreate:
        compose.commands['down'](compose, args)
    # args.no_recreate disables check for changes (which is not implemented)

    podman_command = 'run' if args.detach and not args.no_start else 'create'

    create_pods(compose, args)
    for cnt in compose.containers:
        podman_args = container_to_args(compose, cnt,
            detached=args.detach, podman_command=podman_command)
        compose.podman.run(podman_args)
    if args.no_start or args.detach or args.dry_run: return
    # TODO: handle already existing
    # TODO: if error creating do not enter loop
    # TODO: colors if sys.stdout.isatty()

    threads = []
    for cnt in compose.containers:
        # TODO: remove sleep from podman.run
        thread = Thread(target=compose.podman.run, args=[['start', '-a', cnt['name']]], daemon=True)
        thread.start()
        threads.append(thread)
        time.sleep(1)
    while True:
        for thread in threads:
            thread.join(timeout=1.0)
            if thread.is_alive(): continue
            if args.abort_on_container_exit:
                exit(-1)

@cmd_run(podman_compose, 'down', 'tear down entire stack')
def compose_down(compose, args):
    for cnt in compose.containers:
        compose.podman.run(["stop", "-t=1", cnt["name"]], sleep=0)
    for cnt in compose.containers:
        compose.podman.run(["rm", cnt["name"]], sleep=0)
    for pod in compose.pods:
        compose.podman.run(["pod", "rm", pod["name"]], sleep=0)

@cmd_run(podman_compose, 'run', 'create a container similar to a service to run a one-off command')
def compose_run(compose, args):
    create_pods(compose, args)
    print(args)
    container_names=compose.container_names_by_service[args.service]
    container_name=container_names[0]
    cnt = compose.container_by_name[container_name]
    deps = cnt["_deps"]
    if not args.no_deps:
        # TODO: start services in deps
        pass
    # adjust one-off container options
    name0 = "{}_{}_tmp{}".format(compose.project_name, args.service, random.randrange(0, 65536))
    cnt["name"] = args.name or name0
    if args.entrypoint: cnt["entrypoint"] = args.entrypoint
    if args.user: cnt["user"] = args.user
    if args.workdir: cnt["working_dir"] = args.workdir
    if not args.service_ports:
        for k in ("expose", "publishall", "ports"):
            try: del cnt[k]
            except KeyError: pass
    if args.volume:
        # TODO: handle volumes
        pass
    cnt['tty']=False if args.T else True
    cnt['command']=args.cnt_command
    # run podman
    podman_args = container_to_args(compose, cnt, args.detach)
    if not args.detach:
        podman_args.insert(1, '-i')
        if args.rm:
            podman_args.insert(1, '--rm')
    compose.podman.run(podman_args, sleep=0)

def transfer_service_status(compose, args, action):
    # TODO: handle dependencies, handle creations
    container_names_by_service = compose.container_names_by_service
    targets = []
    for service in args.services:
        if service not in container_names_by_service:
            raise ValueError("unknown service: " + service)
        targets.extend(container_names_by_service[service])
    podman_args=[action]
    timeout=getattr(args, 'timeout', None)
    if timeout is not None:
        podman_args.extend(['-t', "{}".format(timeout)])
    for target in targets:
        compose.podman.run(podman_args+[target], sleep=0)

@cmd_run(podman_compose, 'start', 'start specific services')
def compose_start(compose, args):
    transfer_service_status(compose, args, 'start')

@cmd_run(podman_compose, 'stop', 'stop specific services')
def compose_stop(compose, args):
    transfer_service_status(compose, args, 'start')

@cmd_run(podman_compose, 'restart', 'restart specific services')
def compose_restart(compose, args):
    transfer_service_status(compose, args, 'restart')

###################
# command arguments parsing
###################

@cmd_parse(podman_compose, 'up')
def compose_up_parse(parser):
    parser.add_argument("-d", "--detach", action='store_true',
        help="Detached mode: Run container in the background, print new container name. Incompatible with --abort-on-container-exit.")
    parser.add_argument("--no-color", action='store_true',
        help="Produce monochrome output.")
    parser.add_argument("--quiet-pull", action='store_true',
        help="Pull without printing progress information.")
    parser.add_argument("--no-deps", action='store_true',
        help="Don't start linked services.")
    parser.add_argument("--force-recreate", action='store_true',
        help="Recreate containers even if their configuration and image haven't changed.")
    parser.add_argument("--always-recreate-deps", action='store_true',
        help="Recreate dependent containers. Incompatible with --no-recreate.")
    parser.add_argument("--no-recreate", action='store_true',
        help="If containers already exist, don't recreate them. Incompatible with --force-recreate and -V.")
    parser.add_argument("--no-build", action='store_true',
        help="Don't build an image, even if it's missing.")
    parser.add_argument("--no-start", action='store_true',
        help="Don't start the services after creating them.")
    parser.add_argument("--build", action='store_true',
        help="Build images before starting containers.")
    parser.add_argument("--abort-on-container-exit", action='store_true',
        help="Stops all containers if any container was stopped. Incompatible with -d.")
    parser.add_argument("-t", "--timeout", type=float, default=10,
        help="Use this timeout in seconds for container shutdown when attached or when containers are already running. (default: 10)")
    parser.add_argument("-V", "--renew-anon-volumes", action='store_true',
        help="Recreate anonymous volumes instead of retrieving data from the previous containers.")
    parser.add_argument("--remove-orphans", action='store_true',
        help="Remove containers for services not defined in the Compose file.")
    parser.add_argument('--scale', metavar="SERVICE=NUM", action='append',
        help="Scale SERVICE to NUM instances. Overrides the `scale` setting in the Compose file if present.")
    parser.add_argument("--exit-code-from", metavar='SERVICE', type=str, default=None,
        help="Return the exit code of the selected service container. Implies --abort-on-container-exit.")
    parser.add_argument('services', metavar='SERVICES', nargs='*',
        help='service names to start')

@cmd_parse(podman_compose, 'run')
def compose_run_parse(parser):
    parser.add_argument("-d", "--detach", action='store_true',
        help="Detached mode: Run container in the background, print new container name.")
    parser.add_argument("--name", type=str, default=None,
        help="Assign a name to the container")
    parser.add_argument("--entrypoint", type=str, default=None,
        help="Override the entrypoint of the image.")
    parser.add_argument('-e', metavar="KEY=VAL", action='append',
        help="Set an environment variable (can be used multiple times)")
    parser.add_argument('-l', '--label', metavar="KEY=VAL", action='append',
        help="Add or override a label (can be used multiple times)")
    parser.add_argument("-u", "--user", type=str, default=None,
        help="Run as specified username or uid")
    parser.add_argument("--no-deps", action='store_true',
        help="Don't start linked services")
    parser.add_argument("--rm", action='store_true',
        help="Remove container after run. Ignored in detached mode.")
    parser.add_argument('-p', '--publish', action='append',
        help="Publish a container's port(s) to the host (can be used multiple times)")
    parser.add_argument("--service-ports", action='store_true',
        help="Run command with the service's ports enabled and mapped to the host.")
    parser.add_argument('-v', '--volume', action='append',
        help="Bind mount a volume (can be used multiple times)")
    parser.add_argument("-T", action='store_true',
        help="Disable pseudo-tty allocation. By default `podman-compose run` allocates a TTY.")
    parser.add_argument("-w", "--workdir", type=str, default=None,
        help="Working directory inside the container")
    parser.add_argument('service', metavar='service', nargs=None,
        help='service name')
    parser.add_argument('cnt_command', metavar='command', nargs=argparse.REMAINDER,
        help='command and its arguments')

@cmd_parse(podman_compose, ['stop', 'restart'])
def compose_parse_timeout(parser):
    parser.add_argument("-t", "--timeout",
        help="Specify a shutdown timeout in seconds. ",
        type=float, default=10)

@cmd_parse(podman_compose, ['start', 'stop', 'restart'])
def compose_parse_services(parser):
    parser.add_argument('services', metavar='services', nargs='+',
        help='affected services')

@cmd_parse(podman_compose, 'push')
def compose_push_parse(parser):
    parser.add_argument("--ignore-push-failures", action='store_true',
        help="Push what it can and ignores images with push failures. (not implemented)")
    parser.add_argument('services', metavar='services', nargs='*',
        help='services to push')

@cmd_parse(podman_compose, 'build')
def compose_build_parse(parser):
    parser.add_argument("--pull",
        help="attempt to pull a newer version of the image", action='store_true')
    parser.add_argument("--pull-always",
        help="attempt to pull a newer version of the image, Raise an error even if the image is present locally.", action='store_true')

def main():
    podman_compose.run()

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
"""
Translation with a Sequence to Sequence Network and Attention
*************************************************************
**Author**: `Sean Robertson <https://github.com/spro/practical-pytorch>`_
In this project we will be teaching a neural network to translate from
French to English.
::
    [KEY: > input, = target, < output]
    > il est en train de peindre un tableau .
    = he is painting a picture .
    < he is painting a picture .
    > pourquoi ne pas essayer ce vin delicieux ?
    = why not try that delicious wine ?
    < why not try that delicious wine ?
    > elle n est pas poete mais romanciere .
    = she is not a poet but a novelist .
    < she not not a poet but a novelist .
    > vous etes trop maigre .
    = you re too skinny .
    < you re all alone .
... to varying degrees of success.
This is made possible by the simple but powerful idea of the `sequence
to sequence network <https://arxiv.org/abs/1409.3215>`__, in which two
recurrent neural networks work together to transform one sequence to
another. An encoder network condenses an input sequence into a vector,
and a decoder network unfolds that vector into a new sequence.
.. figure:: /_static/img/seq-seq-images/seq2seq.png
   :alt:
To improve upon this model we'll use an `attention
mechanism <https://arxiv.org/abs/1409.0473>`__, which lets the decoder
learn to focus over a specific range of the input sequence.
**Recommended Reading:**
I assume you have at least installed PyTorch, know Python, and
understand Tensors:
-  https://pytorch.org/ For installation instructions
-  :doc:`/beginner/deep_learning_60min_blitz` to get started with PyTorch in general
-  :doc:`/beginner/pytorch_with_examples` for a wide and deep overview
-  :doc:`/beginner/former_torchies_tutorial` if you are former Lua Torch user
It would also be useful to know about Sequence to Sequence networks and
how they work:
-  `Learning Phrase Representations using RNN Encoder-Decoder for
   Statistical Machine Translation <https://arxiv.org/abs/1406.1078>`__
-  `Sequence to Sequence Learning with Neural
   Networks <https://arxiv.org/abs/1409.3215>`__
-  `Neural Machine Translation by Jointly Learning to Align and
   Translate <https://arxiv.org/abs/1409.0473>`__
-  `A Neural Conversational Model <https://arxiv.org/abs/1506.05869>`__
You will also find the previous tutorials on
:doc:`/intermediate/char_rnn_classification_tutorial`
and :doc:`/intermediate/char_rnn_generation_tutorial`
helpful as those concepts are very similar to the Encoder and Decoder
models, respectively.
And for more, read the papers that introduced these topics:
-  `Learning Phrase Representations using RNN Encoder-Decoder for
   Statistical Machine Translation <https://arxiv.org/abs/1406.1078>`__
-  `Sequence to Sequence Learning with Neural
   Networks <https://arxiv.org/abs/1409.3215>`__
-  `Neural Machine Translation by Jointly Learning to Align and
   Translate <https://arxiv.org/abs/1409.0473>`__
-  `A Neural Conversational Model <https://arxiv.org/abs/1506.05869>`__
**Requirements**
"""
from __future__ import unicode_literals, print_function, division
from io import open
import unicodedata
import string
import re
import random

import torch
import torch.nn as nn
from torch import optim
import torch.nn.functional as F
import csv
from util_seq2seq import Lang, EncoderRNN, DecoderRNN, AttnDecoderRNN
from eval_seq2seq import EvalSeq2Seq

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

######################################################################
# Loading data files
# ==================
#
# The data for this project is a set of many thousands of English to
# French translation pairs.
#
# `This question on Open Data Stack
# Exchange <https://opendata.stackexchange.com/questions/3888/dataset-of-sentences-translated-into-many-languages>`__
# pointed me to the open translation site https://tatoeba.org/ which has
# downloads available at https://tatoeba.org/eng/downloads - and better
# yet, someone did the extra work of splitting language pairs into
# individual text files here: https://www.manythings.org/anki/
#
# The English to French pairs are too big to include in the repo, so
# download to ``data/eng-fra.txt`` before continuing. The file is a tab
# separated list of translation pairs:
#
# ::
#
#     I am cold.    J'ai froid.
#
# .. Note::
#    Download the data from
#    `here <https://download.pytorch.org/tutorial/data.zip>`_
#    and extract it to the current directory.

######################################################################
# Similar to the character encoding used in the character-level RNN
# tutorials, we will be representing each word in a language as a one-hot
# vector, or giant vector of zeros except for a single one (at the index
# of the word). Compared to the dozens of characters that might exist in a
# language, there are many many more words, so the encoding vector is much
# larger. We will however cheat a bit and trim the data to only use a few
# thousand words per language.
#
# .. figure:: /_static/img/seq-seq-images/word-encoding.png
#    :alt:
#
#

######################################################################
# We'll need a unique index per word to use as the inputs and targets of
# the networks later. To keep track of all this we will use a helper class
# called ``Lang`` which has word → index (``word2index``) and index → word
# (``index2word``) dictionaries, as well as a count of each word
# ``word2count`` to use to later replace rare words.
#

SOS_token = 0
EOS_token = 1

######################################################################
# To read the data file we will split the file into lines, and then split
# lines into pairs. The files are all English → Other Language, so if we
# want to translate from Other Language → English I added the ``reverse``
# flag to reverse the pairs.
#

def read_requences():
    print("Reading lines...")

    pairs = []
    houstons = []
    # Read the file and split into lines
    with open('data/need_dataset.csv') as dataset_pointer:
        csv_reader = csv.reader(dataset_pointer, delimiter=',')

        maxlen = 0
        max_sentence = ''
        for i, row in enumerate(csv_reader):
            if i < 1:
                continue

            s1 = ' '.join(row[0:(len(row)-1)])
            # s1 = normalizeString(s1)

            s2 = ' '.join(row[(len(row)-1):len(row)])
            # s2 = normalizeString(s2)
            l = len(s2.split())
            if l > maxlen:
                maxlen = l
                max_sentence = s2
            pairs.append([s1, s2])

            loc = row[0]
            if loc == 'houston':
                houstons.append([s1, s2])

    print('max sentence length:', maxlen, '; setence=', max_sentence)
    # Reverse pairs, make Lang instances
    input_lang = Lang("input_sequence")
    output_lang = Lang("output_sequence")

    return input_lang, output_lang, pairs, houstons

######################################################################
# Since there are a *lot* of example sentences and we want to train
# something quickly, we'll trim the data set to only relatively short and
# simple sentences. Here the maximum length is 10 words (that includes
# ending punctuation) and we're filtering to sentences that translate to
# the form "I am" or "He is" etc. (accounting for apostrophes replaced
# earlier).
#

MAX_LENGTH = 24

######################################################################
# The full process for preparing the data is:
#
# -  Read text file and split into lines, split lines into pairs
# -  Normalize text, filter by length and content
# -  Make word lists from sentences in pairs
#

def prepareData():
    input_lang, output_lang, pairs, houstons = read_requences()
    print("Read %s sentence pairs" % len(pairs))
    print("Trimmed to %s sentence pairs" % len(pairs))
    print("Counting words...")
    for pair in pairs:
        input_lang.addSentence(pair[0])
        output_lang.addSentence(pair[1])
    print("Counted words:")
    print(input_lang.name, input_lang.n_words)
    print(output_lang.name, output_lang.n_words)
    return input_lang, output_lang, pairs, houstons

######################################################################
# Training the Model
# ------------------
#
# To train we run the input sentence through the encoder, and keep track
# of every output and the latest hidden state. Then the decoder is given
# the ``<SOS>`` token as its first input, and the last hidden state of the
# encoder as its first hidden state.
#
# "Teacher forcing" is the concept of using the real target outputs as
# each next input, instead of using the decoder's guess as the next input.
# Using teacher forcing causes it to converge faster but `when the trained
# network is exploited, it may exhibit
# instability <http://minds.jacobs-university.de/sites/default/files/uploads/papers/ESNTutorialRev.pdf>`__.
#
# You can observe outputs of teacher-forced networks that read with
# coherent grammar but wander far from the correct translation -
# intuitively it has learned to represent the output grammar and can "pick
# up" the meaning once the teacher tells it the first few words, but it
# has not properly learned how to create the sentence from the translation
# in the first place.
#
# Because of the freedom PyTorch's autograd gives us, we can randomly
# choose to use teacher forcing or not with a simple if statement. Turn
# ``teacher_forcing_ratio`` up to use more of it.
#

teacher_forcing_ratio = 0.5

def train(input_tensor, target_tensor, encoder: EncoderRNN, decoder: AttnDecoderRNN, encoder_optimizer, decoder_optimizer, criterion, max_length=MAX_LENGTH):
    encoder_hidden = encoder.initHidden(device=device)

    encoder_optimizer.zero_grad()
    decoder_optimizer.zero_grad()

    input_length = input_tensor.size(0)
    target_length = target_tensor.size(0)

    encoder_outputs = torch.zeros(max_length, encoder.hidden_size, device=device)

    loss = 0

    for ei in range(input_length):
        encoder_output, encoder_hidden = encoder(
            input_tensor[ei], encoder_hidden)
        encoder_outputs[ei] = encoder_output[0, 0]

    decoder_input = torch.tensor([[SOS_token]], device=device)

    decoder_hidden = encoder_hidden

    use_teacher_forcing = True if random.random() < teacher_forcing_ratio else False

    if use_teacher_forcing:
        # Teacher forcing: Feed the target as the next input
        for di in range(target_length):
            decoder_output, decoder_hidden, decoder_attention = decoder(
                decoder_input, decoder_hidden, encoder_outputs)
            loss += criterion(decoder_output, target_tensor[di])
            decoder_input = target_tensor[di]  # Teacher forcing

    else:
        # Without teacher forcing: use its own predictions as the next input
        for di in range(target_length):
            decoder_output, decoder_hidden, decoder_attention = decoder(
                decoder_input, decoder_hidden, encoder_outputs)
            topv, topi = decoder_output.topk(1)
            decoder_input = topi.squeeze().detach()  # detach from history as input

            loss += criterion(decoder_output, target_tensor[di])
            if decoder_input.item() == EOS_token:
                break

    loss.backward()

    encoder_optimizer.step()
    decoder_optimizer.step()

    return loss.item() / target_length

######################################################################
# This is a helper function to print time elapsed and estimated time
# remaining given the current time and progress %.
#

import time
import math

def asMinutes(s):
    m = math.floor(s / 60)
    s -= m * 60
    return '%dm %ds' % (m, s)

def timeSince(since, percent):
    now = time.time()
    s = now - since
    es = s / (percent)
    rs = es - s
    return '%s (- %s)' % (asMinutes(s), asMinutes(rs))

def save_model(encoder: EncoderRNN, decoder: AttnDecoderRNN, encoder_optimizer, decoder_optimizer, iteration=None, loss=None):
    # save encoer model
    encoder_state = {'epoch': iteration, 'state_dict': encoder.state_dict(),
                     'optimizer': encoder_optimizer.state_dict(), 'loss': loss, }
    torch.save(encoder_state, 'data/model/encoder_' + str(iteration) + '_' + str(loss) + ".tar")

    # save decoder model
    decoder_state = {'epoch': iteration, 'state_dict': decoder.state_dict(),
                     'optimizer': decoder_optimizer.state_dict(), 'loss': loss, }
    torch.save(decoder_state, 'data/model/decoder_' + str(iteration) + '_' + str(loss) + ".tar")
######################################################################
# The whole training process looks like this:
#
# -  Start a timer
# -  Initialize optimizers and criterion
# -  Create set of training pairs
# -  Start empty losses array for plotting
#
# Then we call ``train`` many times and occasionally print the progress (%
# of examples, time so far, estimated time) and average loss.
#

def trainIters(encoder: EncoderRNN, decoder: AttnDecoderRNN, n_iters, print_every=1000, plot_every=100, learning_rate=0.01):

    global eval_seq

    start = time.time()
    plot_losses = []
    print_loss_total = 0  # Reset every print_every
    plot_loss_total = 0  # Reset every plot_every

    encoder_optimizer = optim.SGD(encoder.parameters(), lr=learning_rate)
    decoder_optimizer = optim.SGD(decoder.parameters(), lr=learning_rate)
    training_pairs = [eval_seq.tensorsFromPair(random.choice(pairs))
                      for i in range(n_iters)]
    criterion = nn.NLLLoss()

    min_loss = 100000
    for iter in range(1, n_iters + 1):
        training_pair = training_pairs[iter - 1]
        input_tensor = training_pair[0]
        target_tensor = training_pair[1]

        loss = train(input_tensor, target_tensor, encoder,
                     decoder, encoder_optimizer, decoder_optimizer, criterion)
        print_loss_total += loss
        plot_loss_total += loss

        # if min_loss > loss:
        #     print("saving model at: iter=", iter, "loss=", loss)
        #     save_model(encoder, decoder, encoder_optimizer, decoder_optimizer, iteration=iter, loss=loss)
        #     min_loss = loss

        if iter % print_every == 0:
            print_loss_avg = print_loss_total / print_every
            print_loss_total = 0
            print('%s (%d %d%%) %.4f' % (timeSince(start, iter / n_iters),
                                         iter, iter / n_iters * 100, print_loss_avg))

        if iter % plot_every == 0:
            plot_loss_avg = plot_loss_total / plot_every
            plot_losses.append(plot_loss_avg)
            plot_loss_total = 0

    filename = 'loss_' + str(n_iters)
    with open('data/model/' + filename + '.csv', 'w') as loss_file:
        loss_writer = csv.writer(loss_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
        for l in plot_losses:
            loss_writer.writerow([l])

        print("Done writing loss file")
    eval_seq.showPlot(plot_losses, force_show=False, save_figure=True, figure_filename='data/model/' + filename + '.png')

######################################################################
# Plotting results
# ----------------
#
# Plotting is done with matplotlib, using the array of loss values
# ``plot_losses`` saved while training.
#

######################################################################
# Training and Evaluating
# =======================
#
# With all these helper functions in place (it looks like extra work, but
# it makes it easier to run multiple experiments) we can actually
# initialize a network and start training.
#
# Remember that the input sentences were heavily filtered. For this small
# dataset we can use relatively small networks of 256 hidden nodes and a
# single GRU layer. After about 40 minutes on a MacBook CPU we'll get some
# reasonable results.
#
# .. Note::
#    If you run this notebook you can train, interrupt the kernel,
#    evaluate, and continue training later. Comment out the lines where the
#    encoder and decoder are initialized and run ``trainIters`` again.
#

hidden_size = 256

input_lang, output_lang, pairs, houston = prepareData()
print(random.choice(pairs))

encoder1 = EncoderRNN(input_lang.n_words, hidden_size).to(device)
attn_decoder1 = AttnDecoderRNN(hidden_size, output_lang.n_words, dropout_p=0.1).to(device)

eval_seq = EvalSeq2Seq(input_lang=input_lang, output_lang=output_lang, pairs=pairs, encoder=encoder1,
                       attn_decoder=attn_decoder1, device=device)

# trainIters(encoder1, attn_decoder1, 75000, print_every=5000)
trainIters(encoder1, attn_decoder1, 250000, print_every=1000)

# eval_seq.evaluateRandomly(encoder1, attn_decoder1)
eval_seq.evaluate_data(encoder1, attn_decoder1, houston, 'data/model')

# output_words, attentions = evaluate(
#     encoder1, attn_decoder1, "je suis trop froid .")
# plt.matshow(attentions.numpy())


# -*- coding: utf-8 -*-
import scrapy
import re
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from scrapy.http import Request
from urlparse import urlparse
from factcheck.items import FactcheckItem
from bs4 import BeautifulSoup

class FactSpider(CrawlSpider):
    name = 'factcheck'

    allowed_domains = ['news.naver.com']
    '''
    start_urls = ['http://news.naver.com/main/list.nhn?oid=023&mid=sec&mode=LPOD&date=20160304'] #sbstv

    rules = (
        Rule(LinkExtractor(allow=r'023&mid=sec&mode=LPOD&date=20160304'), callback='parse_list', follow=True),
    )
    '''
    #snac url http:// 없이 들어옴

    def __init__(self, *a, **kw):

        self.start_urls = ['http://news.naver.com/main/election/president2017/factcheck/index.nhn?type=NEW&page=1']
        FactSpider.rules = (Rule(LinkExtractor(allow=r'type=NEW&page='), callback='parse_list', follow=True),)

        super(FactSpider, self).__init__(*a, **kw)

    def parse_list(self, response):
#//*[@id="factcheck_page"]/div[3]/div[1]/div[1]/div[2]/p
        factItem = FactcheckItem()
        item_list = response.xpath("//div[@class='fact_lst _tab_NEW']/div[@class='section']")
        #self.log("is it here?  ")
        #self.logger.debug(len(item_list))

        for item in item_list:

            name = item.xpath(".//span[@class='name']/text()").extract_first()
            action = item.xpath(".//h4[@class='tit']/text()").extract_first()
            title = item.xpath(".//div[@class='info_area _rcount']/p/text()").extract_first()
            result = item.xpath(".//dl[@class='chk_lst']/dd/text()")

            self.logger.info(name)
            self.logger.info(action)
            self.logger.info(action.split()[-1])
            self.logger.info(title)
            self.logger.info(result.extract())

            factItem['name'] = name
            factItem['action'] = action.split()[-1]
            factItem['title'] = title
            factItem['result'] = result.extract()

            yield factItem
'''

        item_list = response.xpath("//ul[contains(@class,'type06_headline') or contains(@class,'type06')]/li/dl/dt[not(@class='photo')]")

        for item in item_list:
            detail_url = item.xpath("a/@href")[0].extract().strip()
            detail_urls = ''.join(detail_url)
            request = Request(detail_urls, callback=self.parse_detail)
            yield request

    def parse_detail(self, response):

        if response.status == 200:
            if(response.url.find("entertain") == -1 and response.url.find("sports") == -1 ):
                naver_url = response.xpath("//meta[@property='og:url']/@content")[0].extract()
                naver_section = response.xpath("//meta[@property='me2:category2']/@content").extract_first()
                detail_content = response.xpath("//div[@class='content']")
                datetime = detail_content.xpath("//span[@class='t11']/text()").extract_first().strip()
                #comment_cnt = detail_content.xpath("//span[@class='lo_txt']/text()").extract_first().strip()
                comment_cnt = 0
                #origin_url = detail_content.xpath("//div[@class='article_btns']/a/@href").extract_first().strip()
                origin_url = detail_content.xpath("//a[@class='btn_artialoriginal']/@href").extract_first().strip()

                vod = detail_content.xpath("//div[@class='vod_area']").extract()
                origin_content = detail_content.xpath("//div[@id='articleBodyContents']").extract_first().strip()

                article = BeautifulSoup(origin_content, 'lxml')
                # kill all script and style elements
                for script in article(["script", "style"]):
                    script.extract()    # rip it out

                for br in article(["br"]):
                    br.replace_with("\n")
                # get text
                #text = article.get_text()
                article_text = article.getText()

                image_count = len(article.find_all("img"))
                vod_count = len(vod)

                if (origin_url.startswith('http')):
                    yield Request(origin_url, callback=self.parse_origin, meta={'naver_section':naver_section, 'naver_url':naver_url,'datetime':datetime, 'img_cnt':image_count, 'vod_cnt':vod_count, 'cmt_cnt': comment_cnt, 'article':article_text})

    def parse_origin(self, response):
        newsItem = NewscrawlItem()
        parsed_url = urlparse(response.url)
        domain_name = '{uri.netloc}'.format(uri=parsed_url)

        if response.status == 200:
            newsItem['naver_url'] = response.meta['naver_url']
            newsItem['naver_section'] = response.meta['naver_section']
            newsItem['date'] = response.meta['datetime'].split()[0]
            newsItem['time'] = response.meta['datetime'].split()[1]
            newsItem['image'] = response.meta['img_cnt']
            newsItem['vod'] = response.meta['vod_cnt']
            newsItem['comments'] = response.meta['cmt_cnt']
            newsItem['article'] = response.meta['article']
            title = response.xpath("//meta[@property='og:title']/@content")
            #title = response.xpath("//div[contains(@class,'article_title02') or contains(@class, 'articleTit') or contains(@class, 'article_title') or contains(@class, 'artiTitle')]/h1/text()")

            if(len(title)>0):
                newsItem['title'] = title.extract_first()
            else :
                newsItem['title'] = response.xpath("//title/text()").extract_first()

            response_url = response.xpath("//meta[@property='og:url']/@content")
            if(len(response_url) > 0 ):
                newsItem['org_url'] = response_url[0].extract()
            else :
                newsItem['org_url'] = response.url

            #//*[@id="container"]/div[2]/p[1]
            author_items = []
            if(domain_name == "news.chosun.com"):
                author_items = response.xpath("//li[@id='j1']/a/text()").extract()
            elif(domain_name == "biz.chosun.com"):
                author_items = response.xpath("//li[@id='j1']/text()").extract()

            authors = []
            if(len(author_items) > 0):
                for author_item in author_items:
                    authors.append(author_item.strip())

                if(len(authors)>1):
                    newsItem['author'] = ' '.join(authors)
                else:
                    newsItem['author'] = authors[0]

            section = ""
            if(domain_name == "news.chosun.com"):
                title_text = response.xpath('//title/text()').extract_first()
                title_list = title_text.split('-')
                section = title_list[-1]

                section = section.split('>')[0].strip()

                #javascript_codes = response.xpath('//body/script/text()').extract()

                #for code in javascript_codes:
                #    section = re.findall(r"C1_rp.*?=\s*(.*?);", code, re.DOTALL | re.MULTILINE)
            elif(domain_name == "biz.chosun.com"):
                section = domain_name.split('.')[0]
                #response.xpath('//div[@class="art_title_2011"]/dl/dt/a/text()').extract_first()
            else:
                section = domain_name.split('.')[0]
                #self.logger.info("non section url   " + response.url)

            #self.logger.info(section)

            if(len(section) > 0):
                newsItem['section'] = section

            yield newsItem
    '''


'''
Знайти добуток всіх елементів масиву цілих чисел, менших 0. Розмірність
масиву - 10. Заповнення масиву здійснити за допомогою клавіатури.
Постолович А.С. 122-В
'''
from functools import reduce
import numpy as np

A = np.zeros(10) #створюємо пустий масив
z = [] #створюємо пустий список
for i in range(10):
    A[i] = int(input('Введіть значення:')) #заповнюємо масив значенями
    if A[i] < 0:
        z.append(A[i]) #
z_mult = reduce(lambda x, y: x*y, z) #знаходимо добуток всіх чисел у списку
print(f"Добуток всіх від'ємних чисел - {z_mult}") #виводимо добуток всіх чисел у списку


import pickle
import numpy as np
import io
import time
import zlib

a = np.array(range(100000))
b = np.array(range(200000, 300000))
file_obj = io.BytesIO()

start = time.time()
file_obj.write(zlib.compress(pickle.dumps({'foo': a, 'bar': b})))
print('pickle dump: %.04f s' % (time.time() - start))

start = time.time()
p = pickle.loads(zlib.decompress(file_obj.getvalue()))
p['foo']
p['bar']
print('pickle load: %.04f s' % (time.time() - start))

print('pickled size: %s' % len(file_obj.getvalue()))

file_obj = io.BytesIO()
start = time.time()
file_obj.seek(0)
np.savez_compressed(file_obj, foo=a, bar=b)
print('np dump: %.04f s' % (time.time() - start))

start = time.time()
file_obj.seek(0)
npz = np.load(file_obj)
npz['foo']
npz['bar']

print('np load: %.04f s' % (time.time() - start))
print('np size: %s' % len(file_obj.getvalue()))


#!/usr/bin/python

import cv2
import numpy as np
import sys

# img1 = cv2.imread('maskout.bmp')
# img1 = img1.astype(np.float32)
# img2 = cv2.imread('maskout_-15.bmp')
# img2 = img2.astype(np.float32)

img1 = cv2.imread('IMG_7094_half.JPG')
img1 = img1.astype(np.float32)
img2 = cv2.imread('screen_1920x1080_4.png')
img2 = img2.astype(np.float32)

mapper = cv2.reg_MapperGradAffine()
mappPyr = cv2.reg_MapperPyramid(mapper)

resMap = mappPyr.calculate(img1, img2)
mapShift = cv2.reg.MapTypeCaster_toAffine(resMap)

print(mapShift.getLinTr())
print(mapShift.getShift())


import nltk
from nltk.tree import ParentedTree
from nltk.tokenize import RegexpTokenizer
from nltk.sentiment.vader import SentimentIntensityAnalyzer

class ChunkParser(nltk.ChunkParserI):
    def __init__(self, train_sents):
        train_data = [[(t, c) for w, t, c in nltk.chunk.tree2conlltags(sent)] for sent in train_sents]
        self.tagger = nltk.TrigramTagger(train_data)

    def parse(self, sentence):
        pos_tags = [pos for (word, pos) in sentence]
        tagged_pos_tags = self.tagger.tag(pos_tags)
        chunktags = [chunktag for (pos, chunktag) in tagged_pos_tags]
        conlltags = [(word, pos, chunktag) for ((word, pos), chunktag) in zip(sentence, chunktags)]
        return nltk.chunk.conlltags2tree(conlltags)

with open('./drug_abst.txt') as fp:
    text = fp.readlines()
    string = ''.join(text)
    sentences = nltk.sent_tokenize(string)
    sid = SentimentIntensityAnalyzer()

    neg_count = 0
    pos_count = 0
    neu_count = 0

    for sentence in sentences:
        print(sentence)
        ss = sid.polarity_scores(sentence)
        if ss['neg'] > ss['pos']:
            neg_count += 1

        else:
            pos_count += 1

    print('POS: {}, NEG: {}'.format(pos_count, neg_count))

    """
    sentences = nltk.sent_tokenize(string)
    tokens = []
    for sent in sentences:
        pos = nltk.pos_tag(nltk.word_tokenize(sent))
        tokens.append(pos)
    train_sents = nltk.corpus.conll2000.chunked_sents('train.txt', chunk_types=['NP'])
    # print(train_sents)
    NPChunker = ChunkParser(train_sents)
    foo = NPChunker.parse(tokens[0])
    foo = ParentedTree.convert(foo)
    for t in foo.subtrees():
        if t.label() == 'NP':
            print(t)
    """


# coding=utf8

from bs4 import BeautifulSoup
from operator import itemgetter
import base64, Commands, Config, importlib, Logger, praw, re, requests, Soaker, socket, ssl, sys, time, types, URLInfo

class IRC:
    def __init__(self):
        Config.config(self)
        self.activeDict = {}
        for channel in self.info['CHAN']:
            self.activeDict[channel] = {}
        Commands.redditAPI(self)
        Soaker.Soaker(self)
        self.Connect()
        self.fLog = {}
        self.Main()

    def Connect(self):
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.connect((self.info['HOST'], int(self.info['PORT'])))
        self.irc = ssl.wrap_socket(sock)
        connectMSG = []
        if self.info['SASL']:
            self.ircSend('CAP LS')
        self.ircSend('NICK %s' % self.info['NICK'])
        self.ircSend('USER %s %s %s :%s' % (self.info['NICK'], self.info['NICK'], self.info['NICK'], self.info['NAME']))
        self.ircSend('JOIN %s' % ','.join(self.info['CHAN']))

    def Main(self):
        while True:
            rawdata = self.irc.recv(4096)
            try:
                data = rawdata.decode('utf-8')
            except UnicodeDecodeError:
                try:
                    data = rawdata.decode('cp1252')
                except UnicodeDecodeError:
                    try:
                        data = rawdata.decode('iso8859-1')
                    except:
                        continue
            lines = str(data).split('\n')
            for line in lines:
                if len(line) < 1:
                    continue

                Log = Logger.interpret(line)
                self.fLog[Log['time']] = Log

                # reply to pings
                if Log['command'] == 'PING':
                    self.ircSend('PONG :%s' % Log['trail'][0])
                    continue

                # SASL
                if self.info['SASL']:
                    if Log['command'] == 'CAP':
                        if Log['parameters'] [0] == '*' and Log['parameters'][1] == 'LS':
                            self.ircSend('CAP REQ :%s' % ' '.join(Log['trail']))
                            continue
                        if Log['parameters'] [1] == 'ACK':
                            self.ircSend('AUTHENTICATE PLAIN')
                            continue
                    if Log['command'] == 'AUTHENTICATE' and Log['parameters'][0] == '+':
                        sasl_token = '\0'.join([self.info['NICK'], self.info['NICK'], self.info['PASS']])
                        self.ircSend('AUTHENTICATE %s' % base64.b64encode(sasl_token.encode('utf-8')).decode('utf-8'))
                        continue
                    if Log['command'] == '903':
                        self.ircSend('CAP END')
                        self.ircSend('JOIN %s' % ','.join(self.info['CHAN']))
                        continue

                # checks when identified with nickserv
                if Log['command'] == 'NOTICE' and Log['nick'] == 'NickServ':
                    if len(Log['trail']) > 3:
                        if 'registered' in Log['trail'][3]:
                            self.ircSend('PRIVMSG NickServ :identify %s' % self.info['PASS'])
                            continue
                        if Log['trail'][3] == 'identified':
                            self.ircSend('JOIN %s' % ','.join(self.info['CHAN']))
                            continue

                # checks for INVITE received
                if Log['command'] == 'INVITE' and Log['parameters'][0] == self.info['NICK']:
                    if Log['trail'][0] not in self.info['CHAN']:
                        self.info['CHAN'].append(Log['trail'][0])
                        self.updateFile()
                        self.ircSend('JOIN %s' % Log['trail'][0])

                # checks channel join
                if Log['command'] == 'JOIN':
                    if Log['nick'] == self.info['NICK']:
                        self.activeDict[Log['parameters'][0]] = {}
                        self.ircSend('WHO %s %%na' % Log['parameters'][0])
                    elif Log['nick'] not in self.activeDict[Log['parameters'][0]]:
                        self.activeDict[Log['parameters'][0]][Log['nick']] = 0
                        if Log['nick'] not in list(self.userDict.values()):
                            self.ircSend('WHOIS %s' % Log['nick'])
                    continue

                # checks nick change
                if Log['command'] == 'NICK':
                    if Log['nick'] == self.info['NICK']:
                        self.info['NICK'] = Log['trail'][0]
                    else:
                        for channels in self.info['CHAN']:
                            if Log['nick'] in self.activeDict[channels]:
                                self.activeDict[channels][Log['trail'][0]] = self.activeDict[channels][Log['nick']]
                                del self.activeDict[channels][Log['nick']]
                        self.ircSend('WHOIS %s' % Log['trail'][0])
                    continue

                # parses NAMES result
                if str(Log['command']) == '353' and len(Log['parameters']) > 2:
                    if Log['parameters'][-1] not in self.activeDict:
                        self.activeDict[Log['parameters'][-1]] = {}
                    for names in Log['trail']:
                        names = names.lstrip('@+')
                        if names != self.info['NICK']:
                            self.activeDict[Log['parameters'][-1]][names] = 0
                    continue

                # parses WHOIS result
                if (str(Log['command']) == '330' or str(Log['command']) == '354') and len(Log['parameters']) > 2:
                    if Log['parameters'][2] != str(0):
                        if Log['parameters'][2] not in self.userDict:
                            self.userDict[Log['parameters'][2]] = []
                        if Log['parameters'][1] not in self.userDict[Log['parameters'][2]] and Log['parameters'][1] != self.info['NICK']:
                            self.userDict[Log['parameters'][2]].append(Log['parameters'][1])
                        self.updateFile()
                    continue

                # updates active list if user leaves
                if Log['command'] == 'PART':
                    if Log['nick'] in self.activeDict[Log['parameters'][0]]:
                        del self.activeDict[Log['parameters'][0]][Log['nick']]
                    continue
                if Log['command'] == 'QUIT':
                    for channels in self.info['CHAN']:
                        if Log['nick'] in self.activeDict[channels]:
                            del self.activeDict[channels][Log['nick']]
                    continue

                # checks when PRIVMSG received
                if Log['command'] == 'PRIVMSG':
                    if Log['context'] not in self.info['UNLOAD'].keys():
                        self.info['UNLOAD'][Log['context']] = []

                    # builds last spoke list
                    if Log['context'] not in self.activeDict:
                        self.activeDict[Log['context']] = {}
                    self.activeDict[Log['context']][Log['nick']] = Log['time']
                    validList = []
                    for group in self.userDict.values():
                        validList.extend(group)
                    if Log['nick'] not in validList and Log['cap'] == '+':
                        self.ircSend('WHOIS %s' % Log['nick'])

                    if Log['host'] in self.info['OWNER'] + self.info['SUDOER']:
                        if len(Log['trail']) > 1:
                            def imports():
                                for name, val in globals().items():
                                    if isinstance(val, types.ModuleType):
                                        yield val.__name__
                            try:
                                if Log['trail'][0].lower() == '!load':
                                    if Log['trail'][1] in self.info['UNLOAD'][Log['context']]:
                                        self.info['UNLOAD'][Log['context']].remove(Log['trail'][1])
                                        self.PRIVMSG(Log['context'],'Module \'%s\' loaded' % Log['trail'][1])

                                if Log['trail'][0].lower() == '!unload':
                                    if Log['trail'][1] in list(imports()) and Log['trail'][1] not in self.info['UNLOAD'][Log['context']]:
                                        self.info['UNLOAD'][Log['context']].append(Log['trail'][1])
                                        self.PRIVMSG(Log['context'],'Module \'%s\' unloaded' % Log['trail'][1])
                                        self.updateFile()

                                if Log['trail'][0].lower() == '!reload':
                                    module = __import__(Log['trail'][1])
                                    if Log['trail'][1] == 'IRCbot':
                                        self.ircSend('QUIT')
                                        self.irc.close()
                                    if Log['trail'][1] in list(imports()):
                                        importlib.reload(module)
                                        self.PRIVMSG(Log['context'],'Module \'%s\' reloaded' % Log['trail'][1])
                            except Exception as e:
                                print(e)

                        if Log['trail'][0] == '!quit':
                            self.ircSend('QUIT')
                            self.irc.close()
                            sys.exit('User exited')

                    if 'Soaker' not in self.info['UNLOAD'][Log['context']]:
                        Soaker.Handler(self, Log)

                    if 'Commands' not in self.info['UNLOAD'][Log['context']]:
                        Commands.Handler(self, Log)

                    if 'URLInfo' not in self.info['UNLOAD'][Log['context']]:
                        URLInfo.Handler(self, Log)

    def updateFile(self):
        with open('nwobot.conf', 'r') as file:
            oldConf = eval(file.read())
        with open('users', 'r') as file:
            oldUser = eval(file.read())
        if self.info != oldConf:
            with open('nwobot.conf', 'w+') as file:
                file.write(str(self.info))
        if self.userDict != oldUser:
            with open('users', 'w+') as file:
                file.write(str(self.userDict))

    def listActive(self, chan, minutes = 10, caller = None, full = False, exclude = []):
        return Logger.listActive(self, chan, minutes, caller, full, exclude)

    current_milli_time = lambda: int(round(time.time() * 1000))

    def PRIVMSG(self, context, message):
        self.ircSend('PRIVMSG %s :%s' % (context, message))

    def ircSend(self, message):
        print('%s %s' % (time.strftime('%H:%M:%S', time.gmtime(time.time())), message))
        self.irc.send(bytes('%s\r\n' % message, 'UTF-8'))

IRC()


# -*- coding: utf-8 -*-\
'''
    unit.cloud_config_test
    ~~~~~~~~~~~~~~~~~~~~~~

    Configuration related unit testing

    :codeauthor: :email:`Pedro Algarvio (pedro@algarvio.me)`
    :copyright: © 2013 by the SaltStack Team, see AUTHORS for more details.
    :license: Apache 2.0, see LICENSE for more details.
'''

# Import python libs
import os
import shutil
import tempfile

# Import 3rd-party libs
import yaml

# Import salt testing libs
from salttesting import TestCase
from salttesting.helpers import ensure_in_syspath
ensure_in_syspath('../')

# Import salt libs
import salt.utils
import integration
from salt import config as cloudconfig

class CloudConfigTestCase(TestCase):

    def test_load_cloud_config_from_environ_var(self):
        original_environ = os.environ.copy()

        tempdir = tempfile.mkdtemp()
        try:
            env_root_dir = os.path.join(tempdir, 'foo', 'env')
            os.makedirs(env_root_dir)
            env_fpath = os.path.join(env_root_dir, 'config-env')

            salt.utils.fopen(env_fpath, 'w').write(
                'root_dir: {0}\n'
                'log_file: {1}\n'.format(env_root_dir, env_fpath)
            )

            os.environ['SALT_CLOUD_CONFIG'] = env_fpath
            # Should load from env variable, not the default configuration file
            config = cloudconfig.cloud_config('/etc/salt/cloud')
            self.assertEqual(config['log_file'], env_fpath)
            os.environ.clear()
            os.environ.update(original_environ)

            root_dir = os.path.join(tempdir, 'foo', 'bar')
            os.makedirs(root_dir)
            fpath = os.path.join(root_dir, 'config')
            salt.utils.fopen(fpath, 'w').write(
                'root_dir: {0}\n'
                'log_file: {1}\n'.format(root_dir, fpath)
            )
            # Let's set the environment variable, yet, since the configuration
            # file path is not the default one, ie, the user has passed an
            # alternative configuration file form the CLI parser, the
            # environment variable will be ignored.
            os.environ['SALT_CLOUD_CONFIG'] = env_fpath
            config = cloudconfig.cloud_config(fpath)
            self.assertEqual(config['log_file'], fpath)
        finally:
            # Reset the environ
            os.environ.clear()
            os.environ.update(original_environ)

            if os.path.isdir(tempdir):
                shutil.rmtree(tempdir)

    def test_deploy_search_path_as_string(self):
        temp_conf_dir = os.path.join(integration.TMP, 'issue-8863')
        config_file_path = os.path.join(temp_conf_dir, 'cloud')
        deploy_dir_path = os.path.join(temp_conf_dir, 'test-deploy.d')
        try:
            for directory in (temp_conf_dir, deploy_dir_path):
                if not os.path.isdir(directory):
                    os.makedirs(directory)

            default_config = cloudconfig.cloud_config(config_file_path)
            default_config['deploy_scripts_search_path'] = deploy_dir_path
            with salt.utils.fopen(config_file_path, 'w') as cfd:
                cfd.write(yaml.dump(default_config))

            default_config = cloudconfig.cloud_config(config_file_path)

            # Our custom deploy scripts path was correctly added to the list
            self.assertIn(
                deploy_dir_path,
                default_config['deploy_scripts_search_path']
            )

            # And it's even the first occurrence as it should
            self.assertEqual(
                deploy_dir_path,
                default_config['deploy_scripts_search_path'][0]
            )
        finally:
            if os.path.isdir(temp_conf_dir):
                shutil.rmtree(temp_conf_dir)

    def test_includes_load(self):
        '''cloud.{providers,profiles}.d directories are loaded even if not directly passed'''
        config_path = os.path.join(integration.FILES, 'conf', 'cloud')
        config = cloudconfig.cloud_config(config_path)
        self.assertIn('ec2-config', config['providers'])
        self.assertIn('Ubuntu-13.04-AMD64', config['profiles'])

if __name__ == '__main__':
    from salttesting.parser import run_testcase
    run_testcase(CloudConfigTestCase)


import z3
import ast
import logging
#from pyState import z3Helpers
from pyObjectManager.Int import Int
from pyObjectManager.Real import Real
from pyObjectManager.BitVec import BitVec
from pyObjectManager.List import List
from pyObjectManager.Ctx import Ctx
from pyObjectManager.String import String
from pyObjectManager.Char import Char
import pyState

logger = logging.getLogger("ObjectManager")

CTX_GLOBAL = 0
CTX_RETURNS = 1

class ObjectManager:
    """
    Object Manager will keep track of objects. Generally, Objects will be variables such as ints, lists, strings, etc.
    """

    def __init__(self,variables=None,returnObjects=None,state=None):
        self.variables = {CTX_GLOBAL: Ctx(CTX_GLOBAL), CTX_RETURNS: Ctx(CTX_RETURNS)} if variables is None else variables
        self.returnObjects = returnObjects if returnObjects is not None else {}

        if state is not None:
            self.setState(state)

    def setState(self,state):
        """
        This is a bit strange, but State won't copy correctly due to Z3, so I need to bypass this a bit by setting State each time I copy
        """
        assert type(state) == pyState.State

        self.state = state
        for ctx in self.variables:
            self.variables[ctx].setState(state)

    def newCtx(self,ctx):
        """
        Sets up a new context (ctx)
        """
        assert ctx is not None

        self.variables[ctx] = Ctx(ctx)
        self.variables[ctx].setState(self.state)

    def setVar(self,varName,ctx,var):
        """
        Input:
            varName = variable name (i.e.: 'x')
            ctx = Context to set for
            var = variable object of type pyObjectManager.X
        Action:
            Sets variable to the input (var) object
        Returns:
            Nothing
        """
        assert type(varName) is str
        assert type(ctx) is int
        assert type(var) in [Int, Real, BitVec, List]

        self.variables[ctx][varName] = var

    def getVar(self,varName,ctx,varType=None,kwargs=None,softFail=None):
        """
        Input:
            varName = name of variable to get
            ctx = Context for variable
            (optional) varType = Class type of variable (ex: pyObjectManager.Int)
            (optional) kwargs = args needed to instantiate variable
            (optional) softFail = True/False, should raise an exception if getVar fails. Default is False
        Action:
            Find appropriate variable object, creating one if necessary
        Returns:
            pyObjectManager object for given variable (i.e.: pyObjectManager.Int)
        """

        # Attempt to return variable
        assert type(varName) is str
        assert type(ctx) is int
        assert varType in [None, Int, Real, BitVec, List, String, Char]

        softFail = False if softFail is None else softFail
        create = False
        count = None

        # Check that we already have this variable defined
        if varName in self.variables[ctx]:

            # Check the type of the var is correct
            if varType is not None:

                # If the variable type is different or it's settings are different, we need to create a new object
                if type(self.variables[ctx][varName]) is not varType or not self.variables[ctx][varName]._isSame(**kwargs if kwargs is not None else {}):
                    create = True
                    # Re-using variable names is BAD!
                    count = self.variables[ctx][varName].count + 1

            # If we can just return the current one, let's do it
            if not create:
                return self.variables[ctx][varName]

        # Looks like we need to create a var
        if varType == None:
            if not softFail:
                err = "getVar: Need to create '{0}' but no type information given".format(varName)
                logger.error(err)
                raise Exception(err)
            else:
                # We're soft failing, just return None
                return None

        # Make the var
        self.variables[ctx][varName] = varType(varName=varName,ctx=ctx,count=count,state=self.state,**kwargs if kwargs is not None else {})

        return self.variables[ctx][varName]

    def getParent(self,key,haystack=None):
        """
        Returns the parent object for any given object by recursively searching.
        """
        # TODO: This might get to be a long search if there are a lot of variables...

        haystack = self.variables if haystack is None else haystack

        if type(haystack) in [dict, Ctx]:
            for k,v in haystack.items():
                if v == key:
                    return haystack
                elif type(v) in [dict, List, Ctx]:
                    p = self.getParent(key,v)
                    if p:
                        return p
        elif isinstance(haystack,List):
            for v in haystack:
                if v == key:
                    return haystack
                elif type(v) in [dict,List]:
                    p = self.getParent(key,v)
                    if p:
                        return p

    def copy(self):
        """
        Return a copy of the Object Manager
        """

        return ObjectManager(
            variables = {key:self.variables[key].copy() for key in self.variables},
            returnObjects = {key:self.returnObjects[key].copy() for key in self.returnObjects}
        )


import logging
import re
from steelfig.models.mysql import Mysql
from steelfig.models.account import Account

logger = logging.getLogger(__name__)
class Wishlist(object):

    def __init__(self, event_id=None, account_id=None):
        self.id = None
        self.event_id = event_id
        self.account_id = account_id

        if event_id and account_id:
            self.id = self.get_id_from_attendee(event_id, account_id)

    def get_id_from_attendee(self, event_id, account_id):
        db = Mysql()
        cursor = db.get_cursor()
        cursor.execute('''
            SELECT wishlists.id
            FROM wishlists
            JOIN attendees ON wishlists.attendee_id = attendees.id
            WHERE attendees.event_id = %s
            AND attendees.account_id = %s
            LIMIT 1
        ''', (event_id, account_id))
        data = cursor.fetchone()
        return False if not data else int(data.get('id'))

    def get_my_items(self):
        data = []
        if self.id:
            db = Mysql()
            cursor = db.get_cursor()
            cursor.execute('''
                SELECT id, name, url, price, sort, comments, created_at, updated_at
                FROM wishlist_items
                WHERE wishlist_id = %s
                ORDER BY sort ASC
            ''', (self.id,))
            data = cursor.fetchall()
        return {'event_wishlist_items': data}

    def get_buddies_items(self):
        data = []
        if self.id:
            db = Mysql()
            cursor = db.get_cursor()
            cursor.execute('''
                SELECT id, name, url, price, sort, comments, is_purchased, created_at, updated_at
                FROM wishlist_items
                WHERE wishlist_id = %s
                ORDER BY sort ASC
            ''', (self.id,))
            data = cursor.fetchall()
        return {'event_wishlist_items': data}

    def set_item(self, account_id, params, id=None):
        db = Mysql()
        cursor = db.get_cursor()
        if not params.get('name'):
            raise Exception('no name found')

        # Convert money to integer to avoid floating point crazyness
        price = str(params.get('price'))
        if price:
            price = re.sub(r'[^0-9\.]', '', price)
            price = "{:10.2f}".format(float(price))
            price = float(price) * 100

        # If url doesn't start with http, prepend it
        url = params.get('url')
        if url and not url.startswith('http'):
            url = 'http://' + url

        cursor.execute('''
            CALL set_wishlist_item(
                if(%s = '', NULL, %s),
                if(%s = '', NULL, %s),
                if(%s = '', NULL, %s),
                %s, %s, %s,
                if(%s = '', NULL, %s),
                %s, %s, %s)
        ''', (id or '', id or '',
              account_id, account_id,
              params.get('event_id', ''), params.get('event_id'),
              params.get('name', ''),
              params.get('barcode', ''),
              url,
              price, price,
              params.get('is_purchased', 0),
              params.get('sort', 0),
              params.get('comments')))
        data = cursor.fetchone()
        return {'event_wishlist_item': data}

    def delete_item(self, account_id, wishlist_item_id):
        db = Mysql()
        cursor = db.get_cursor()
        cursor.execute('''
            DELETE FROM wishlist_items
            WHERE id = %s
            AND wishlist_id IN (
                SELECT wishlists.id
                FROM wishlists
                JOIN attendees ON wishlists.attendee_id = attendees.id
                WHERE attendees.account_id = %s)
            LIMIT 1
        ''', (wishlist_item_id, account_id))
        return True


# -*- coding: utf-8 -*-

import os
import sys
import dlib
import cv2
import glob

detector = dlib.simple_object_detector("projects/dlib-learning/custom-train/detector.svm")

cap = cv2.VideoCapture(0)
if not cap.isOpened():
    print('cannot open camera')
    exit(0)

while True:
    ret,img = cap.read()
    if not ret:
        print('cannot grab image from camera')
        exit(0)
    b, g, r = cv2.split(img)
    img2 = cv2.merge([r, g, b])
    dets = detector(img2)
    print("Number of faces detected: {}".format(len(dets)))
    cv2.namedWindow("camera", cv2.WINDOW_AUTOSIZE)
    cv2.imshow("camera", img)
    for index, face in enumerate(dets):
        print('face {}; left {}; top {}; right {}; bottom {}'.format(index, face.left(), face.top(), face.right(), face.bottom()))

        left = face.left()
        top = face.top()
        right = face.right()
        bottom = face.bottom()
        cv2.rectangle(img, (left, top), (right, bottom), (0, 255, 0), 3)
    cv2.imshow("camera", img)
    key = cv2.waitKey(10)
    if key == 27:
        break

cv2.destroyAllWindows()