import sys
import dpkt
import time
import socket
import logging
import threading
import subprocess
import diverterbase
import netfilterqueue
from linutil import *
from diverterbase import *
from collections import namedtuple
from netfilterqueue import NetfilterQueue

class PacketHandler:
    """Used to encapsulate common patterns in packet hooks."""

    def __init__(self, pkt, diverter, label, callbacks3, callbacks4):
        self.logger = logging.getLogger('Diverter')

        self.pkt = pkt
        self.diverter = diverter  # Relies on Diverter for certain operations
        self.label = label
        self.callbacks3 = callbacks3
        self.callbacks4 = callbacks4

        self.raw = self.pkt.get_payload()
        self.ipver = ((ord(self.raw[0]) & 0xf0) >> 4)
        self.hdr, self.proto = self.diverter.parse_pkt[self.ipver](self.ipver,
                                                                   self.raw)

    def handle_pkt(self):
        """Generic packet hook.

        1.) Common prologue:
            A.) Unconditionally Write unmangled packet to pcap
            B.) Parse IP packet

        2.) Call layer 3 (network) callbacks...

        3.) Parse higher-layer protocol (TCP, UDP) for port numbers

        4.) Call layer 4 (transport) callbacks...

        5.) Common epilogue:
            A.) If the packet headers have been modified:
                i.) Double-write the mangled packet to the pcap for SSL
                    decoding purposes
                ii.) Update the packet payload with NetfilterQueue
            B.) Accept the packet with NetfilterQueue
        """

        # 1A: Unconditionally write unmangled packet to pcap
        self.diverter.write_pcap(self.hdr.pack())

        if (self.hdr, self.proto) == (None, None):
            self.logger.warning('%s: Failed to parse IP packet' % (self.label))
        else:
            proto_name = self.diverter.handled_protocols.get(self.proto)

            self.diverter.pdebug(DGENPKT, '%s %s' % (self.label,
                                 self.diverter.hdr_to_str(proto_name,
                                 self.hdr)))

            # 1B: Parse IP packet (actually done in ctor)
            self.src_ip = socket.inet_ntoa(self.hdr.src)
            self.dst_ip = socket.inet_ntoa(self.hdr.dst)

            # 2: Call layer 3 (network) callbacks
            for cb in self.callbacks3:
                # These debug outputs are useful for figuring out which
                # callback is responsible for an exception that was masked by
                # python-netfilterqueue's global callback.
                self.diverter.pdebug(DCB, 'Calling %s' % (cb))

                cb(self.label, self.hdr, self.ipver, self.proto, proto_name,
                   self.src_ip, self.dst_ip)

                self.diverter.pdebug(DCB, '%s finished' % (cb))

            if proto_name:

                if len(self.callbacks4):
                    # 3: Parse higher-layer protocol
                    self.sport = self.hdr.data.sport
                    self.dport = self.hdr.data.dport
                    self.skey = self.diverter.gen_endpoint_key(proto_name,
                                                               self.src_ip,
                                                               self.sport)
                    self.dkey = self.diverter.gen_endpoint_key(proto_name,
                                                               self.dst_ip,
                                                               self.dport)

                    pid, comm = self.diverter.linux_get_pid_comm_by_endpoint(
                        self.ipver, proto_name, self.src_ip, self.sport)

                    if proto_name == 'UDP':
                        fmt = '| {label} {proto} | {pid:>6} | {comm:<8} | {src:>15}:{sport:<5} | {dst:>15}:{dport:<5} | {length:>5} | {flags:<11} | {seqack:<35} |'
                        logline = fmt.format(
                                label=self.label,
                                proto=proto_name,
                                pid=pid,
                                comm=comm,
                                src=self.src_ip,
                                sport=self.sport,
                                dst=self.dst_ip,
                                dport=self.dport,
                                length=len(self.raw),
                                flags='',
                                seqack='',
                            )
                        self.diverter.pdebug(DGENPKTV, logline)

                    elif proto_name == 'TCP':
                        tcp = self.hdr.data
                        # Interested in:
                        # SYN
                        # SYN,ACK
                        # ACK
                        # PSH
                        # FIN
                        syn = (tcp.flags & dpkt.tcp.TH_SYN) != 0
                        ack = (tcp.flags & dpkt.tcp.TH_ACK) != 0
                        fin = (tcp.flags & dpkt.tcp.TH_FIN) != 0
                        psh = (tcp.flags & dpkt.tcp.TH_PUSH) != 0
                        rst = (tcp.flags & dpkt.tcp.TH_RST) != 0

                        sa = 'Seq=%d, Ack=%d' % (tcp.seq, tcp.ack)
                        f = []
                        if rst:
                            f.append('RST')
                        if syn:
                            f.append('SYN')
                        if ack:
                            f.append('ACK')
                        if fin:
                            f.append('FIN')
                        if psh:
                            f.append('PSH')

                        fmt = '| {label} {proto} | {pid:>6} | {comm:<8} | {src:>15}:{sport:<5} | {dst:>15}:{dport:<5} | {length:>5} | {flags:<11} | {seqack:<35} |'
                        logline = fmt.format(
                                label=self.label,
                                proto=proto_name,
                                pid=pid,
                                comm=comm,
                                src=self.src_ip,
                                sport=self.sport,
                                dst=self.dst_ip,
                                dport=self.dport,
                                length=len(self.raw),
                                flags=','.join(f),
                                seqack=sa,
                            )
                        self.diverter.pdebug(DGENPKTV, logline)

                    if ((not (self.diverter.pdebug_level & DGENPKTV)) and
                        pid and (pid != self.diverter.pid)):
                        self.logger.info('  pid:  %d name: %s' %
                                         (pid, comm if comm else 'Unknown'))

                    hdr_latest = self.hdr
                    modified = False

                    # 4: Layer 4 (Transport layer) callbacks
                    for cb in self.callbacks4:
                        # These debug outputs are useful for figuring out which
                        # callback is responsible for an exception that was
                        # masked by python-netfilterqueue's global callback.
                        self.diverter.pdebug(DCB, 'Calling %s' % (cb))

                        hdr_mod = cb(self.label, pid, comm, self.ipver,
                                     hdr_latest, proto_name,
                                     self.src_ip, self.sport, self.skey,
                                     self.dst_ip, self.dport, self.dkey)

                        if hdr_mod:
                            hdr_latest = hdr_mod
                            modified = True

                        self.diverter.pdebug(DCB, '%s finished' % (cb))

                    if modified:
                        # 5Ai: Double write mangled packets to represent changes
                        # made by FakeNet-NG while still allowing SSL decoding
                        self.diverter.write_pcap(hdr_latest.pack())

                        # 5Aii: Finalize changes with nfq
                        self.pkt.set_payload(hdr_latest.pack())
            else:
                self.diverter.pdebug(DGENPKT, '%s: Not handling protocol %s' %
                                     (self.label, self.proto))

        # 5B: NF_ACCEPT
        self.pkt.accept()

class Diverter(DiverterBase, LinUtilMixin):

    def __init__(self, diverter_config, listeners_config, ip_addrs,
                 logging_level=logging.INFO):
        self.init_base(diverter_config, listeners_config, ip_addrs,
                       logging_level)

        self.init_linux_mixin()
        self.init_diverter_linux()

    def init_diverter_linux(self):
        """Linux-specific Diverter initialization."""
        # String list configuration item that is specific to the Linux
        # Diverter, will not be parsed by DiverterBase, and needs to be
        # accessed as an array in the future.
        slists = ['linuxredirectnonlocal', 'DebugLevel']
        self.reconfigure(portlists=[], stringlists=slists)

        dbg_lvl = 0
        if self.is_configured('DebugLevel'):
            for label in self.getconfigval('DebugLevel'):
                label = label.upper()
                if label == 'OFF':
                    dbg_lvl = 0
                    break
                if not label in DLABELS_INV:
                    self.logger.warning('No such DebugLevel as %s' % (label))
                else:
                    dbg_lvl |= DLABELS_INV[label]
        self.set_debug_level(dbg_lvl, DLABELS)

        # SingleHost vs MultiHost mode
        mode = 'SingleHost'  # Default
        self.single_host_mode = True
        if self.is_configured('networkmode'):
            mode = self.getconfigval('networkmode')
            available_modes = ['singlehost', 'multihost']

            # Constrain argument values
            if mode.lower() not in available_modes:
                self.logger.error('NetworkMode must be one of %s' %
                                  (available_modes))
                sys.exit(1)

            # Adjust previously assumed mode if user specifies MultiHost
            if mode.lower() == 'multihost':
                self.single_host_mode = False

        if self.single_host_mode:
            while True:
                prompt = ('You acknowledge that SingleHost mode on Linux is ' +
                          'experimental and not functionally complete? ' +
                          '[Y/N] ')
                acknowledgement = raw_input(prompt)
                okay = ['y', 'yes', 'yeah', 'sure', 'okay', 'whatever']
                nope = ['n', 'no', 'nah', 'nope']
                if acknowledgement.lower() in okay:
                    self.logger.info('Okay, we\'ll take it for a spin!')
                    break
                elif acknowledgement.lower() in nope:
                    self.logger.error('User opted out of crowd-sourced ' +
                                      'alpha testing program ;-)')
                    sys.exit(1)

        self.logger.info('Running in %s mode' % (mode))

        self.parse_pkt = dict()
        self.parse_pkt[4] = self.parse_ipv4
        self.parse_pkt[6] = self.parse_ipv6

        self.nfqueues = list()

        self.handled_protocols = {
            dpkt.ip.IP_PROTO_TCP: 'TCP',
            dpkt.ip.IP_PROTO_UDP: 'UDP',
        }

        # Track iptables rules not associated with any nfqueue object
        self.rules_added = []

        # Manage logging of foreign-destined packets
        self.nonlocal_ips_already_seen = []
        self.log_nonlocal_only_once = True

        # Port forwarding table, for looking up original unbound service ports
        # when sending replies to foreign endpoints that have attempted to
        # communicate with unbound ports. Allows fixing up source ports in
        # response packets. Similar to the `sessions` member of the Windows
        # Diverter implementation.
        self.port_fwd_table = dict()
        self.port_fwd_table_lock = threading.Lock()

        # Track conversations that will be ignored so that e.g. an RST response
        # from a closed port does not erroneously trigger port forwarding and
        # silence later replies to legitimate clients.
        self.ignore_table = dict()
        self.ignore_table_lock = threading.Lock()

        # IP forwarding table, for looking up original foreign destination IPs
        # when sending replies to local endpoints that have attempted to
        # communicate with other machines e.g. via hard-coded C2 IP addresses.
        self.ip_fwd_table = dict()
        self.ip_fwd_table_lock = threading.Lock()

        # NOTE: Constraining cache size via LRU or similar is a non-requirement
        # due to the short anticipated runtime of FakeNet-NG. If you see your
        # FakeNet-NG consuming large amounts of memory, contact your doctor to
        # find out if Ctrl+C is right for you.

        # The below callbacks are configured to be efficiently executed by a
        # PacketHandler object within the nonlocal, incoming, and outgoing
        # packet hooks installed by the start method.

        # Network layer callbacks for nonlocal-destined packets
        #
        # Log nonlocal-destined packets and ICMP packets before they are NATted
        # to localhost
        self.nonlocal_net_cbs = [self.check_log_nonlocal, self.check_log_icmp]

        # Network and transport layer callbacks for incoming packets
        #
        # IP redirection fix-ups are only for SingleHost mode.
        self.incoming_net_cbs = []
        self.incoming_trans_cbs = [self.maybe_redir_port]
        if self.single_host_mode:
            self.incoming_trans_cbs.append(self.maybe_fixup_srcip)

        # Network and transport layer callbacks for outgoing packets.
        #
        # Must scan for nonlocal packets in the output hook and at the network
        # layer (regardless of whether supported protocols like TCP/UDP can be
        # parsed) when using the SingleHost mode of FakeNet-NG. Note that if
        # this check were performed when FakeNet-NG is operating in MultiHost
        # mode, every response packet generated by a listener and destined for
        # a remote host would erroneously be sent for potential logging as
        # nonlocal host communication. ICMP logging is performed for outgoing
        # packets in SingleHost mode because this will allow logging of the
        # original destination IP address before it was mangled to redirect the
        # packet to localhost.
        self.outgoing_net_cbs = []
        if self.single_host_mode:
            self.outgoing_net_cbs.append(self.check_log_nonlocal)
            self.outgoing_net_cbs.append(self.check_log_icmp)

        self.outgoing_trans_cbs = [self.maybe_fixup_sport]

        # IP redirection is only for SingleHost mode
        if self.single_host_mode:
            self.outgoing_trans_cbs.append(self.maybe_redir_ip)

    def start(self):
        self.logger.info('Starting Linux Diverter...')

        if not self.check_privileged():
            self.logger.error('The Linux Diverter requires administrative ' +
                              'privileges')
            sys.exit(1)

        ret = self.linux_capture_iptables()
        if ret != 0:
            sys.exit(1)

        if self.is_set('linuxflushiptables'):
            self.linux_flush_iptables()
        else:
            self.logger.warning('LinuxFlushIptables is disabled, this may ' +
                                'result in unanticipated behavior depending ' +
                                'upon what rules are already present')

        hookspec = namedtuple('hookspec', ['chain', 'table', 'callback'])

        callbacks = list()

        # If you are considering adding or moving hooks that mangle packets,
        # see the section of docs/internals.md titled Explaining Hook Location
        # Choices for an explanation of how to avoid breaking the Linux NAT
        # implementation.
        if not self.single_host_mode:
            callbacks.append(hookspec('PREROUTING', 'raw',
                                      self.handle_nonlocal))

        callbacks.append(hookspec('INPUT', 'mangle', self.handle_incoming))
        callbacks.append(hookspec('OUTPUT', 'raw', self.handle_outgoing))

        nhooks = len(callbacks)

        self.pdebug(DNFQUEUE, ('Discovering the next %d available NFQUEUE ' +
                    'numbers') % (nhooks))
        qnos = self.linux_get_next_nfqueue_numbers(nhooks)
        if len(qnos) != nhooks:
            self.logger.error('Could not procure a sufficient number of ' +
                              'netfilter queue numbers')
            sys.exit(1)

        self.pdebug(DNFQUEUE, 'Next available NFQUEUE numbers: ' + str(qnos))

        self.pdebug(DNFQUEUE, 'Enumerating queue numbers and hook ' +
                    'specifications to create NFQUEUE objects')
        self.nfqueues = list()
        for qno, hk in zip(qnos, callbacks):
            self.pdebug(DNFQUEUE, ('Creating NFQUEUE object for chain %s / ' +
                        'table %s / queue # %d => %s') % (hk.chain, hk.table,
                        qno, str(hk.callback)))
            q = LinuxDiverterNfqueue(qno, hk.chain, hk.table, hk.callback)
            self.nfqueues.append(q)
            ok = q.start()
            if not ok:
                self.logger.error('Failed to start NFQUEUE for %s' % (str(q)))
                self.stop()
                sys.exit(1)

        if self.single_host_mode and self.is_set('fixgateway'):
            if not self.linux_get_default_gw():
                self.linux_set_default_gw()

        if self.single_host_mode and self.is_set('modifylocaldns'):
            self.linux_modifylocaldns_ephemeral()

        if self.is_configured('linuxflushdnscommand') and self.single_host_mode:
            cmd = self.getconfigval('linuxflushdnscommand')
            ret = subprocess.call(cmd.split())
            if ret != 0:
                self.logger.error(
                'Failed to flush DNS cache. Local machine may use cached DNS results.')

        if self.is_configured('linuxredirectnonlocal'):
            self.pdebug(DMISC, 'Processing LinuxRedirectNonlocal')
            specified_ifaces = self.getconfigval('linuxredirectnonlocal')
            self.pdebug(DMISC, 'Processing linuxredirectnonlocal on ' +
                        'interfaces: %s' % (specified_ifaces))
            ok, rules = self.linux_iptables_redir_nonlocal(specified_ifaces)

            # Irrespective of whether this failed, we want to add any
            # successful iptables rules to the list so that stop() will be able
            # to remove them using linux_remove_iptables_rules().
            self.rules_added += rules

            if not ok:
                self.logger.error('Failed to process LinuxRedirectNonlocal')
                self.stop()
                sys.exit(1)

        ok, rule = self.linux_redir_icmp()
        if not ok:
            self.logger.error('Failed to redirect ICMP')
            self.stop()
            sys.exit(1)

        self.rules_added.append(rule)

    def stop(self):
        self.logger.info('Stopping Linux Diverter...')

        self.pdebug(DNFQUEUE, 'Notifying NFQUEUE objects of imminent stop')
        for q in self.nfqueues:
            q.stop_nonblocking()

        self.pdebug(DIPTBLS, 'Removing iptables rules not associated with any ' +
                    'NFQUEUE object')
        self.linux_remove_iptables_rules(self.rules_added)

        for q in self.nfqueues:
            self.pdebug(DNFQUEUE, 'Stopping NFQUEUE for %s' % (str(q)))
            q.stop()

        if self.pcap:
            self.pdebug(DMISC, 'Closing pcap file %s' % (self.pcap_filename))
            self.pcap.close()  # Only after all queues are stopped

        self.logger.info('Stopped Linux Diverter')

        if self.single_host_mode and self.is_set('modifylocaldns'):
            self.linux_restore_local_dns()

        self.linux_restore_iptables()

    def getOriginalDestPort(self, orig_src_ip, orig_src_port, proto):
        """Return original destination port, or None if it was not redirected
        """

        orig_src_key = self.gen_endpoint_key(proto, orig_src_ip, orig_src_port)
        self.port_fwd_table_lock.acquire()

        try:
            if orig_src_key in self.port_fwd_table:
                return self.port_fwd_table[orig_src_key]

            return None
        finally:
            self.port_fwd_table_lock.release()

    def handle_nonlocal(self, pkt):
        """Handle comms sent to IP addresses that are not bound to any adapter.

        This allows analysts to observe when malware is communicating with
        hard-coded IP addresses in MultiHost mode.
        """
        h = PacketHandler(pkt, self, 'handle_nonlocal', self.nonlocal_net_cbs,
                [])
        h.handle_pkt()

    def handle_incoming(self, pkt):
        """Incoming packet hook.

        Specific to incoming packets:
        5.) If SingleHost mode:
            a.) Conditionally fix up source IPs to support IP forwarding for
                otherwise foreign-destined packets
        4.) Conditionally mangle destination ports to implement port forwarding
            for unbound ports to point to the default listener

        No return value.
        """
        h = PacketHandler(pkt, self, 'handle_incoming', self.incoming_net_cbs,
                self.incoming_trans_cbs)
        h.handle_pkt()

    def handle_outgoing(self, pkt):
        """Outgoing packet hook.

        Specific to outgoing packets:
        4.) If SingleHost mode:
            a.) Conditionally log packets destined for foreign IP addresses
                (the corresponding check for MultiHost mode is called by
                handle_nonlocal())
            b.) Conditionally mangle destination IPs for otherwise foreign-
                destined packets to implement IP forwarding
        5.) Conditionally fix up mangled source ports to support port
            forwarding

        No return value.
        """
        h = PacketHandler(pkt, self, 'handle_outgoing', self.outgoing_net_cbs,
                self.outgoing_trans_cbs)
        h.handle_pkt()

    def parse_ipv4(self, ipver, raw):
        hdr = dpkt.ip.IP(raw)
        if hdr.hl < 5:
            return (None, None)  # An IP header length less than 5 is invalid
        return hdr, hdr.p

    def parse_ipv6(self, ipver, raw):
        hdr = dpkt.ip6.IP6(raw)
        return hdr, hdr.nxt

    def gen_endpoint_key(self, proto_name, ip, port):
        """e.g. 192.168.19.132:tcp/3030"""
        return str(ip) + ':' + str(proto_name) + '/' + str(port)

    def check_log_icmp(self, label, hdr, ipver, proto, proto_name, src_ip,
                       dst_ip):
        if proto == dpkt.ip.IP_PROTO_ICMP:
            self.logger.info('ICMP type %d code %d %s' % (
                hdr.data.type, hdr.data.code, self.hdr_to_str(None, hdr)))

        return None

    def check_log_nonlocal(self, label, hdr, ipver, proto, proto_name, src_ip,
                           dst_ip):
        if dst_ip not in self.ip_addrs[ipver]:
            self._maybe_log_nonlocal(hdr, ipver, proto, dst_ip)

        return None

    def _maybe_log_nonlocal(self, hdr, ipver, proto, dst_ip):
        """Conditionally log packets having a foreign destination.

        Each foreign destination will be logged only once if the Linux
        Diverter's internal log_nonlocal_only_once flag is set. Otherwise, any
        foreign destination IP address will be logged each time it is observed.
        """
        proto_name = self.handled_protocols.get(proto)

        self.pdebug(DNONLOC, 'Nonlocal %s' %
                    (self.hdr_to_str(proto_name, hdr)))

        first_sighting = (dst_ip not in self.nonlocal_ips_already_seen)

        if first_sighting:
            self.nonlocal_ips_already_seen.append(dst_ip)

        # Log when a new IP is observed OR if we are not restricted to
        # logging only the first occurrence of a given nonlocal IP.
        if first_sighting or (not self.log_nonlocal_only_once):
            self.logger.info(
                'Received nonlocal IPv%d datagram destined for %s' %
                (ipver, dst_ip))

    def check_should_ignore(self, pid, comm, ipver, hdr, proto_name, src_ip,
                            sport, dst_ip, dport):
        # SingleHost mode checks
        if self.single_host_mode:
            if comm:
                if comm in self.blacklist_processes:
                    self.pdebug(DIGN, ('Ignoring %s packet from process %s ' +
                                'in the process blacklist.') % (proto_name,
                                comm))
                    self.pdebug(DIGN, '  %s' %
                                (self.hdr_to_str(proto_name, hdr)))
                    return True

                elif (len(self.whitelist_processes) and (comm not in
                      self.whitelist_processes)):
                    self.pdebug(DIGN, ('Ignoring %s packet from process %s ' +
                                'not in the process whitelist.') % (proto_name,
                                comm))
                    self.pdebug(DIGN, '  %s' %
                                (self.hdr_to_str(proto_name, hdr)))
                    return True

                # Check per-listener blacklisted process list
                elif ((proto_name in self.port_process_blacklist) and
                        (dport in self.port_process_blacklist[proto_name])):
                    # If program DOES match blacklist
                    if comm in self.port_process_blacklist[proto_name][dport]:
                        self.pdebug(DIGN, ('Ignoring %s request packet from ' +
                                    'process %s in the listener process ' +
                                    'blacklist.') % (proto_name, comm))
                        self.pdebug(DIGN, '  %s' %
                                    (self.hdr_to_str(proto_name, hdr)))

                    return True

                # Check per-listener whitelisted process list
                elif ((proto_name in self.port_process_whitelist) and
                        (dport in self.port_process_whitelist[proto_name])):
                    # If program does NOT match whitelist
                    if not comm in self.port_process_whitelist[proto_name][dport]:
                        self.pdebug(DIGN, ('Ignoring %s request packet from ' +
                                    'process %s not in the listener process ' +
                                    'whitelist.') % (proto_name, comm))
                        self.pdebug(DIGN, '  %s' %
                                    (self.hdr_to_str(proto_name, hdr)))
                        return True

        # MultiHost mode checks
        else:
            pass  # None as of yet

        # Checks independent of mode

        if set(self.blacklist_ports[proto_name]).intersection([sport, dport]):
            self.pdebug(DIGN, 'Forwarding blacklisted port %s packet:' %
                        (proto_name))
            self.pdebug(DIGN, '  %s' % (self.hdr_to_str(proto_name, hdr)))
            return True

        global_host_blacklist = self.getconfigval('hostblacklist')
        if global_host_blacklist and dst_ip in global_host_blacklist:
            self.pdebug(DIGN, ('Ignoring %s packet to %s in the host ' +
                        'blacklist.') % (proto_name, dst_ip))
            self.pdebug(DIGN, '  %s' % (self.hdr_to_str(proto_name, hdr)))
            return True

        if ((proto_name in self.port_host_whitelist) and
                (dport in self.port_host_whitelist[proto_name])):
            # If host does NOT match whitelist
            if not dst_ip in self.port_host_whitelist[proto_name][dport]:
                self.pdebug(DIGN, ('Ignoring %s request packet to %s not in ' +
                            'the listener host whitelist.') % (proto_name,
                            dst_ip))
                self.pdebug(DIGN, '  %s' % (self.hdr_to_str(proto_name, hdr)))
                return True

        if ((proto_name in self.port_host_blacklist) and
                (dport in self.port_host_blacklist[proto_name])):
            # If host DOES match blacklist
            if dst_ip in self.port_host_blacklist[proto_name][dport]:
                self.pdebug(DIGN, ('Ignoring %s request packet to %s in the ' +
                            'listener host blacklist.') % (proto_name, dst_ip))
                self.pdebug(DIGN, '  %s' % (self.hdr_to_str(proto_name, hdr)))
                return True

        # Duplicated from diverters/windows.py:
        # HACK: FTP Passive Mode Handling
        # Check if a listener is initiating a new connection from a
        # non-diverted port and add it to blacklist. This is done to handle a
        # special use-case of FTP ACTIVE mode where FTP server is initiating a
        # new connection for which the response may be redirected to a default
        # listener.  NOTE: Additional testing can be performed to check if this
        # is actually a SYN packet
        if pid == self.pid:
            if (
                ((dst_ip in self.ip_addrs[ipver]) and
                (not dst_ip.startswith('127.'))) and
                ((src_ip in self.ip_addrs[ipver]) and
                (not dst_ip.startswith('127.'))) and
                (not set([sport, dport]).intersection(self.diverted_ports[proto_name]))
                ):

                self.pdebug(DIGN | DFTP, 'Listener initiated %s connection' %
                            (proto_name))
                self.pdebug(DIGN | DFTP, '  %s' % (self.hdr_to_str(proto_name, hdr)))
                self.pdebug(DIGN | DFTP, '  Blacklisting port %d' % (sport))
                self.blacklist_ports[proto_name].append(sport)

            return True

        return False

    def maybe_redir_ip(self, label, pid, comm, ipver, hdr, proto_name, src_ip,
                       sport, skey, dst_ip, dport, dkey):
        """Conditionally redirect foreign destination IPs to localhost.

        Used only under SingleHost mode.

        Returns:
            None - if unmodified
            dpkt.ip.hdr - if modified
        """
        hdr_modified = None

        if self.check_should_ignore(pid, comm, ipver, hdr, proto_name, src_ip,
                                    sport, dst_ip, dport):
            return hdr_modified  # None

        self.pdebug(DIPNAT, 'Condition 1 test')
        # Condition 1: If the remote IP address is foreign to this system,
        # then redirect it to a local IP address.
        if self.single_host_mode and (dst_ip not in self.ip_addrs[ipver]):
            self.pdebug(DIPNAT, 'Condition 1 satisfied')
            self.ip_fwd_table_lock.acquire()
            try:
                self.ip_fwd_table[skey] = dst_ip

            finally:
                self.ip_fwd_table_lock.release()

            newdst = '127.0.0.1'
            hdr_modified = self.mangle_dstip(hdr, proto_name, dst_ip, newdst)

        else:
            # Delete any stale entries in the IP forwarding table: If the
            # local endpoint appears to be reusing a client port that was
            # formerly used to connect to a foreign host (but not anymore),
            # then remove the entry. This prevents a packet hook from
            # faithfully overwriting the source IP on a later packet to
            # conform to the foreign endpoint's stale connection IP when
            # the host is reusing the port number to connect to an IP
            # address that is local to the FakeNet system.

            self.ip_fwd_table_lock.acquire()
            try:
                if skey in self.ip_fwd_table:
                    self.pdebug(DIPNAT, ' - DELETING ipfwd key entry: ' + skey)
                    del self.ip_fwd_table[skey]
            finally:
                self.ip_fwd_table_lock.release()

        return hdr_modified

    def maybe_fixup_srcip(self, label, pid, comm, ipver, hdr, proto_name,
                          src_ip, sport, skey, dst_ip, dport, dkey):
        """Conditionally fix up the source IP address if the remote endpoint
        had their connection IP-forwarded.

        Check is based on whether the remote endpoint corresponds to a key in
        the IP forwarding table.

        Returns:
            None - if unmodified
            dpkt.ip.hdr - if modified
        """
        hdr_modified = None

        # Condition 4: If the local endpoint (IP/port/proto) combo
        # corresponds to an endpoint that initiated a conversation with a
        # foreign endpoint in the past, then fix up the source IP for this
        # incoming packet with the last destination IP that was requested
        # by the endpoint.
        self.pdebug(DIPNAT, "Condition 4 test: was remote endpoint IP fwd'd?")
        self.ip_fwd_table_lock.acquire()
        try:
            if self.single_host_mode and (dkey in self.ip_fwd_table):
                self.pdebug(DIPNAT, 'Condition 4 satisfied')
                self.pdebug(DIPNAT, ' = FOUND ipfwd key entry: ' + dkey)
                new_srcip = self.ip_fwd_table[dkey]
                hdr_modified = self.mangle_srcip(
                    hdr, proto_name, hdr.src, new_srcip)
            else:
                self.pdebug(DIPNAT, ' ! NO SUCH ipfwd key entry: ' + dkey)
        finally:
            self.ip_fwd_table_lock.release()

        return hdr_modified

    def maybe_redir_port(self, label, pid, comm, ipver, hdr, proto_name,
                         src_ip, sport, skey, dst_ip, dport, dkey):
        hdr_modified = None

        # Get default listener port for this proto, or bail if none
        default = None
        if not proto_name in self.default_listener:
            return hdr_modified  # None
        default = self.default_listener[proto_name]

        # Pre-condition 1: RedirectAllTraffic: Yes
        # NOTE: This only applies to port redirection in the Windows Diverter;
        # IP addresses will be modified by the Windows Diverter when
        # RedirectAllTraffic is disabled. So, the Linux Diverter implementation
        # will follow suit.
        if not self.is_set('redirectalltraffic'):
            self.pdebug(DIGN, 'Ignoring %s packet %s' %
                        (proto_name, self.hdr_to_str(proto_name, hdr)))
            return hdr_modified  # None

        # Pre-condition 1: destination must not be present in port forwarding
        # table (prevents masqueraded ports responding to unbound ports from
        # being mistaken as starting a conversation with an unbound port).
        found = False
        self.port_fwd_table_lock.acquire()
        try:
            # Uses dkey to cross-reference
            found = dkey in self.port_fwd_table
        finally:
            self.port_fwd_table_lock.release()

        if found:
            return hdr_modified  # None

        bound_ports = self.diverted_ports.get(proto_name, [])

        # First, check if this packet is sent from a listener/diverter
        # If so, don't redir for 'Hidden' status because it is already
        # being forwarded from proxy listener to bound/hidden listener
        # Next, check if listener for this port is 'Hidden'. If so, we need to
        # divert it to the proxy as per the Hidden config
        if (dport in bound_ports and pid != self.pid and
                bound_ports[dport] is True):

            #divert to proxy
            hdr_modified = self.mangle_dstport(hdr, proto_name, dport, default)

            # Record the foreign endpoint and old destination port in the port
            # forwarding table
            self.pdebug(DDPFV, ' + ADDING portfwd key entry: ' + skey)
            self.port_fwd_table_lock.acquire()
            try:
                self.port_fwd_table[skey] = dport
            finally:
                self.port_fwd_table_lock.release()

            # Record the altered port for making the ExecuteCmd decision
            dport = default

        # Condition 2: If the packet is destined for an unbound port, then
        # redirect it to a bound port and save the old destination IP in
        # the port forwarding table keyed by the source endpoint identity.

        elif self.decide_redir_port(ipver, proto_name, default, bound_ports,
                                  src_ip, sport, dst_ip, dport):
            self.pdebug(DDPFV, 'Condition 2 satisfied')

            # Post-condition 1: General ignore conditions are not met, or this
            # is part of a conversation that is already being ignored.
            #
            # Placed after the decision to redirect for three reasons:
            # 1.) We want to ensure that the else condition below has a chance
            #     to check whether to delete a stale port forwarding table
            #     entry.
            # 2.) Checking these conditions is, on average, more expensive than
            #     checking if the packet would be redirected in the first
            #     place.
            # 3.) Reporting of packets that are being ignored (i.e. not
            #     redirected), which is integrated into this check, should only
            #     appear when packets would otherwise have been redirected.

            # Is this conversation already being ignored for DPF purposes?
            self.ignore_table_lock.acquire()
            try:
                if dkey in self.ignore_table and self.ignore_table[dkey] == sport:
                    # This is a reply (e.g. a TCP RST) from the
                    # non-port-forwarded server that the non-port-forwarded
                    # client was trying to talk to. Leave it alone.
                    return hdr_modified  # None
            finally:
                self.ignore_table_lock.release()

            if self.check_should_ignore(pid, comm, ipver, hdr, proto_name,
                                        src_ip, sport, dst_ip, dport):
                self.ignore_table_lock.acquire()
                try:
                    self.ignore_table[skey] = dport
                finally:
                    self.ignore_table_lock.release()
                return hdr_modified  # None

            # Record the foreign endpoint and old destination port in the port
            # forwarding table
            self.pdebug(DDPFV, ' + ADDING portfwd key entry: ' + skey)
            self.port_fwd_table_lock.acquire()
            try:
                self.port_fwd_table[skey] = dport
            finally:
                self.port_fwd_table_lock.release()

            hdr_modified = self.mangle_dstport(hdr, proto_name, dport, default)

            # Record the altered port for making the ExecuteCmd decision
            dport = default

        else:
            # Delete any stale entries in the port forwarding table: If the
            # foreign endpoint appears to be reusing a client port that was
            # formerly used to connect to an unbound port on this server,
            # remove the entry. This prevents the OUTPUT or other packet
            # hook from faithfully overwriting the source port to conform
            # to the foreign endpoint's stale connection port when the
            # foreign host is reusing the port number to connect to an
            # already-bound port on the FakeNet system.

            self.delete_stale_port_fwd_key(skey)

        if not (sport in self.sessions and self.sessions[sport] == (dst_ip,
                dport)):
            self.sessions[sport] = (dst_ip, dport)

            if pid and (dst_ip in self.ip_addrs[ipver]):
                cmd = self.build_cmd(proto_name, pid, comm, src_ip,
                                     sport, dst_ip, dport)
                if cmd:
                    self.logger.info('Executing command: %s', cmd)
                    self.execute_detached(cmd)

        return hdr_modified

    def delete_stale_port_fwd_key(self, skey):
        self.port_fwd_table_lock.acquire()
        try:
            if skey in self.port_fwd_table:
                self.pdebug(DDPFV, ' - DELETING portfwd key entry: ' + skey)
                del self.port_fwd_table[skey]
        finally:
            self.port_fwd_table_lock.release()

    def maybe_fixup_sport(self, label, pid, comm, ipver, hdr, proto_name,
                          src_ip, sport, skey, dst_ip, dport, dkey):
        """Conditionally fix up source port if the remote endpoint had their
        connection port-forwarded.

        Check is based on whether the remote endpoint corresponds to a key in
        the port forwarding table.

        Returns:
            None - if unmodified
            dpkt.ip.hdr - if modified
        """
        hdr_modified = None

        # Condition 3: If the remote endpoint (IP/port/proto) combo
        # corresponds to an endpoint that initiated a conversation with an
        # unbound port in the past, then fix up the source port for this
        # outgoing packet with the last destination port that was requested
        # by that endpoint. The term "endpoint" is (ab)used loosely here to
        # apply to UDP host/port/proto combos and any other protocol that
        # may be supported in the future.
        self.pdebug(DDPFV, "Condition 3 test: was remote endpoint port fwd'd?")
        self.port_fwd_table_lock.acquire()
        try:
            if dkey in self.port_fwd_table:
                self.pdebug(DDPFV, 'Condition 3 satisfied: must fix up ' +
                            'source port')
                self.pdebug(DDPFV, ' = FOUND portfwd key entry: ' + dkey)
                new_sport = self.port_fwd_table[dkey]
                hdr_modified = self.mangle_srcport(
                    hdr, proto_name, hdr.data.sport, new_sport)
            else:
                self.pdebug(DDPFV, ' ! NO SUCH portfwd key entry: ' + dkey)
        finally:
            self.port_fwd_table_lock.release()

        return hdr_modified

    def decide_redir_port(self, ipver, proto_name, default_port, bound_ports,
                          src_ip, sport, dst_ip, dport):
        """Decide whether to redirect a port.

        Optimized logic derived by truth table + k-map. See docs/internals.md
        for details.
        """
        # A, B, C, D for easy manipulation; full names for readability only.
        a = src_local = (src_ip in self.ip_addrs[ipver])
        c = sport_bound = sport in (bound_ports)
        d = dport_bound = dport in (bound_ports)

        if self.pdebug_level & DDPFV:
            # Unused logic term not calculated except for debug output
            b = dst_local = (dst_ip in self.ip_addrs[ipver])

            self.pdebug(DDPFV, 'src %s (%s)' %
                        (str(src_ip), ['foreign', 'local'][a]))
            self.pdebug(DDPFV, 'dst %s (%s)' %
                        (str(dst_ip), ['foreign', 'local'][b]))
            self.pdebug(DDPFV, 'sport %s (%sbound)' %
                        (str(sport), ['un', ''][c]))
            self.pdebug(DDPFV, 'dport %s (%sbound)' %
                        (str(dport), ['un', ''][d]))

            def bn(x): return '1' if x else '0'  # Bool -> binary
            self.pdebug(DDPFV, 'abcd = ' + bn(a) + bn(b) + bn(c) + bn(d))

        return (not a and not d) or (not c and not d)

    def mangle_dstip(self, hdr, proto_name, dstip, newdstip):
        """Mangle destination IP for selected outgoing packets."""
        self.pdebug(DIPNAT, 'REDIRECTING %s to IP %s' %
                    (self.hdr_to_str(proto_name, hdr), newdstip))
        hdr.dst = socket.inet_aton(newdstip)
        self._calc_csums(hdr)
        return hdr

    def mangle_srcip(self, hdr, proto_name, src_ip, new_srcip):
        """Mangle source IP for selected incoming packets."""
        self.pdebug(DIPNAT, 'MASQUERADING %s from IP %s' %
                    (self.hdr_to_str(proto_name, hdr), new_srcip))
        hdr.src = socket.inet_aton(new_srcip)
        self._calc_csums(hdr)
        return hdr

    def mangle_dstport(self, hdr, proto_name, dstport, newdstport):
        """Mangle destination port for selected incoming packets."""
        self.pdebug(DDPF, 'REDIRECTING %s to port %d' %
                          (self.hdr_to_str(proto_name, hdr), newdstport))
        hdr.data.dport = newdstport
        self._calc_csums(hdr)
        return hdr

    def mangle_srcport(self, hdr, proto_name, srcport, newsrcport):
        """Mangle source port for selected outgoing packets."""
        self.pdebug(DDPF, 'MASQUERADING %s from port %d' %
                          (self.hdr_to_str(proto_name, hdr), newsrcport))
        hdr.data.sport = newsrcport
        self._calc_csums(hdr)
        return hdr

    def hdr_to_str(self, proto_name, hdr):
        src_ip = socket.inet_ntoa(hdr.src)
        dst_ip = socket.inet_ntoa(hdr.dst)
        if proto_name:
            return '%s %s:%d->%s:%d' % (proto_name, src_ip, hdr.data.sport,
                                        dst_ip, hdr.data.dport)
        else:
            return '%s->%s' % (src_ip, dst_ip)

    def _calc_csums(self, hdr):
        """The roundabout dance of inducing dpkt to recalculate checksums."""
        hdr.sum = 0
        hdr.data.sum = 0
        str(hdr)  # This has the side-effect of invoking dpkt.in_cksum() et al

if __name__ == '__main__':
    logging.basicConfig(format='%(message)s')
    diverterbase.test_redir_logic(Diverter)


from ete3 import PhyloTree, Tree, TreeStyle
from ete3 import add_face_to_node, TextFace, AttrFace, SequenceFace

nw = '(0:2.0, (1:1.0, 2:1.0)3:1.0)4;'
fa = """
>0
A
>1
C
>2
C
"""

t = PhyloTree(nw, alignment=fa, alg_format='fasta', format=1)
ts = TreeStyle()
ts.show_branch_length = False
ts.show_leaf_name = False
ts.draw_guiding_lines = True
ts.draw_aligned_faces_as_table = True
ts.show_scale = False

def my_layout(node):
    #
    # add names to all nodes (not just to leaf nodes)
    # ete3/test/test_treeview/face_rotation.py
    F = TextFace(node.name, tight_text=True)
    add_face_to_node(F, node, column=0, position="branch-right")
    #
    # add branch lengths
    # ete3/treeview/qt4_render.py
    if not node.is_root():
        bl_face = AttrFace("dist", fsize=8, ftype="Arial",
                fgcolor="black", formatter="%0.3g")
        #
        # This is a failed attempt to center the branch length text on the branch.
        #a = 1 # 0 left, 1 center, 2 right
        #bl_face.hz_align = a
        #bl_face.vt_align = a
        #add_face_to_node(bl_face, node, column=0, aligned=True, position="branch-top")
        add_face_to_node(bl_face, node, column=0, position="branch-top")
    #
    # I guess we also have to explicitly add the alignment column
    # if we are overriding the layout function.
    # ete3/treeview/layouts.py : phylogeny(node)
    if hasattr(node, 'sequence'):
        seq_face = SequenceFace(node.sequence, seqtype='nt', fsize=13)
        seq_face.margin_left = 4
        add_face_to_node(seq_face, node, column=1, aligned=True)

ts.layout_fn = my_layout

t.render('treefig.png', tree_style=ts)


import argparse
import matplotlib.pyplot as plt
import matplotlib
import numpy as np
import datetime
import pandas as pd

import json
import seaborn as sns
import matplotlib.pyplot as plt
import os
import matplotlib as mpl

from utils.uniquify import uniquify

def hash_method_plot(cross_play_results, args):
    #plots a graph with average off-diagonal cross-play values

    mean = cross_play_results.mean(axis=1)
    dev = cross_play_results.std(axis=1)

    print('Latex code for table of results:')
    resultstring = ""
    for i, m in enumerate(mean):
        resultstring += "& {:.2f} (\\textpm {:.2f}) ".format(m, dev[i])
    print(resultstring)

    x_vals = 2**np.arange(args.n_different_seeds)

    mpl.rcParams.update({
        "font.size": 9
    })
    mpl.rcParams['pdf.fonttype'] = 42
    mpl.rcParams['ps.fonttype'] = 42

    plt.figure(figsize=(2.9, 2.6))

    ax1 = plt.axes()
    plt.plot(x_vals, mean, '-')
    plt.plot(x_vals, [args.optimum for i in range(cross_play_results.shape[0])], '--')
    plt.fill_between(x_vals, mean - dev, mean + dev, alpha=0.1)

    ax1.set_xscale('log', basex=2)
    ax1.set_xticks(x_vals)
    ax1.get_xaxis().set_major_formatter(matplotlib.ticker.ScalarFormatter())

    plt.title('{}'.format(args.title))
    plt.xlabel('Number of seeds used for tie-breaking')
    plt.ylabel('Avg off-diagonal cross-play value')

    plt.tight_layout()

    if args.save:
        filename = 'results/op_with_tie_breaking/evalrun_{}_op_tie_breaking_plot_{}.pdf'.format(args.run, args.env)
        filename = uniquify(filename)
        plt.savefig(filename)

    plt.show()

def load_cross_play_results(args):

    filename = 'results/sacred/' + args.run + '/config.json'
    print('\n\nloading config from ' + filename)
    with open(filename) as json_file:
        config = json.load(json_file)

    args.env = config["env"]
    args.seed = config["seed"]
    args.n_seeds_per_run = config["n_seeds_per_run"]
    args.hash_function_seeds = config["hash_function_seeds"]

    filename = 'results/sacred/' + args.run + '/info.json'
    print('\n\nloading returns from ' + filename)
    returns = []
    with open(filename) as json_file:
        data = json.load(json_file)
        for dict in data['test_return_mean']:
            returns.append(dict['value'])

    args.n_total_runs = len(returns) // args.n_seeds_per_run ** 2

    #here, each "seed per run" is actually an independent run used to evaluate the hash method

    args.n_different_seeds = args.n_total_runs // len(args.hash_function_seeds)

    cross_play_results = np.zeros((args.n_different_seeds, len(args.hash_function_seeds)))

    print('Number of returns: {}'.format(len(returns)))
    print('Number of independent runs that have been considered: {}'.format(args.n_seeds_per_run))
    print('Number of different amounts of seeds: {}'.format(args.n_different_seeds))
    print('Number of hash function seeds: {}'.format(len(args.hash_function_seeds)))

    for n_seeds in range(args.n_different_seeds):
        for hash_seed in range(len(args.hash_function_seeds)):
            start_index = (n_seeds * len(args.hash_function_seeds) + hash_seed) * args.n_seeds_per_run ** 2
            end_index = (n_seeds * len(args.hash_function_seeds) + hash_seed + 1) * args.n_seeds_per_run ** 2
            return_matrix = np.array(returns[start_index: end_index]).reshape(args.n_seeds_per_run, args.n_seeds_per_run)
            off_diag = return_matrix[~np.eye(return_matrix.shape[0], dtype=bool)]

            print('\n\n-----------\nNumber of seeds: {}'.format(2**n_seeds))
            print('Hash seed index: {}'.format(hash_seed))
            print('off_diagonal_mean:')
            print(off_diag.mean())

            print('off_diagonal_std:')
            print(off_diag.std())

            cross_play_results[n_seeds, hash_seed] = off_diag.mean()

    return cross_play_results, args

if __name__ == '__main__':
    print('\n\n===================================')
    print(datetime.datetime.now())
    # Define the parser
    parser = argparse.ArgumentParser(description='Short sample app')

    parser.add_argument('--run', action="store", dest='run', default=None) #sacred directory of the run generated by op_with_tie_breaking.py
    parser.add_argument('--save', action="store", dest='save', default=True) #whether to save results to file
    parser.add_argument('--title', action="store", dest='title', default='') #title for plot
    parser.add_argument('--optimum', action="store", dest='optimum', default='0.5')

    # Now, parse the command line arguments and store the
    # values in the `args` variable
    args = parser.parse_args()

    args.optimum = float(args.optimum)

    cross_play_results, args = load_cross_play_results(args)

    hash_method_plot(cross_play_results, args)

    print('\n\n')


from __future__ import absolute_import

import json
import requests
import datetime
import logging
import email.utils
from collections import OrderedDict

from django.conf import settings
from django.contrib.auth.models import User
from django.http import HttpResponse, HttpResponseRedirect
from django.views.decorators.csrf import csrf_exempt
from django.template import Template, TemplateDoesNotExist
from django.template.loader import get_template
from django.contrib.sites.models import Site
from django.utils import timezone

logger = logging.getLogger(__name__)

class MailgunException(Exception):
    pass

def address_map(mailgun_data, key, exclude=None):
    if not exclude: exclude = []
    a_map = OrderedDict()
    if key in mailgun_data:
        for a in mailgun_data[key]:
            name, address = email.utils.parseaddr(a)
            if not address in exclude:
                a_map[address.lower()] = a
        # print("mailgun_data[%s] = %s" % (key, mailgun_data[key]))
        # print("exclude = %s" % exclude)
        # print("a_map = %s" % a_map)
    return a_map

def clean_mailgun_data(mailgun_data):
    # Make sure we have what we need
    if not 'to' in mailgun_data or not 'from' in mailgun_data or not 'subject' in mailgun_data:
        raise MailgunException("Mailgun data missing FROM, TO, or SUBJECT!")
    logger.debug("dirty mailgun_data: %s" % mailgun_data)

    # Compile all our addresses
    from_name, from_address = email.utils.parseaddr(mailgun_data["from"])
    to_name, to_address = email.utils.parseaddr(mailgun_data["to"][0])
    exclude = [from_address, to_address]
    bccs = address_map(mailgun_data, "bcc", exclude)
    exclude.extend(bccs.keys())
    # We do not want to remove our first 'to' address
    to_exclude = list(set(exclude))
    to_exclude.remove(to_address)
    tos = address_map(mailgun_data, "to", to_exclude)
    exclude.extend(tos.keys())
    ccs = address_map(mailgun_data, "cc", exclude)

    # Repopulate our data with our clean lists
    mailgun_data["bcc"] = bccs.values()
    mailgun_data["cc"] = ccs.values()
    mailgun_data["to"] = tos.values()

    logger.debug("clean mailgun_data: %s" % mailgun_data)
    return mailgun_data

def inject_list_headers(mailgun_data):
    # Attach some headers: LIST-ID, REPLY-TO, Precedence...
    # Reply-To: list email apparently has some religious debates
    # (http://www.gnu.org/software/mailman/mailman-admin/node11.html)
    # Precedence: list - helps some out of office auto responders know not to send their auto-replies.
    to_name, to_address = email.utils.parseaddr(mailgun_data["to"][0])
    mailgun_data["h:List-Id"] = to_address
    mailgun_data["h:Reply-To"] = to_address
    mailgun_data["h:Precedence"] = "list"

def mailgun_send(mailgun_data, files=None, clean_first=True, inject_list_id=True):
    if clean_first: clean_mailgun_data(mailgun_data)
    if inject_list_id: inject_list_headers(mailgun_data)

    # Make sure nothing goes out if the system is in debug mode
    if settings.DEBUG:
        if not hasattr(settings, 'MAILGUN_DEBUG') or settings.MAILGUN_DEBUG:
            # We will see this message in the mailgun logs but nothing will actually be delivered
            logger.debug("mailgun_send: setting testmode=yes")
            mailgun_data["o:testmode"] = "yes"

    resp = requests.post("https://api.mailgun.net/v2/%s/messages" % settings.MAILGUN_DOMAIN,
                         auth=("api", settings.MAILGUN_API_KEY),
                         data=mailgun_data,
                         files=files
                         )
    logger.debug("Mailgun response: %s" % resp.text)
    return HttpResponse(status=200)

# TODO - move to email?
def send_manage_member(user, subject=None):
    if subject == None:
        subject = "Incomplete Tasks"
    # Adjust the subject if we have a prefix
    if hasattr(settings, "EMAIL_SUBJECT_PREFIX"):
        subject = settings.EMAIL_SUBJECT_PREFIX.strip() + " " + subject.strip()

    subject = "%s - %s" % (subject, user.get_full_name())
    text_content, html_content = get_manage_member_content(user)
    mailgun_data = {"from": settings.EMAIL_ADDRESS,
                    "to": [settings.TEAM_EMAIL_ADDRESS, ],
                    "subject": subject,
                    "text": text_content,
                    "html": html_content,
                    }
    return mailgun_send(mailgun_data, inject_list_id=False)

def render_templates(context, email_key):
    text_content = None
    html_content = None

    try:
        text_template = get_template("email/%s.txt" % email_key)
        if text_template:
            text_content = text_template.render(context)

        html_template = get_template("email/%s.html" % email_key)
        if html_template:
            html_content = html_template.render(context)
    except TemplateDoesNotExist:
        pass

    #logger.debug("text_context: %s" % text_content)
    #logger.debug("html_content: %s" % html_content)
    return (text_content, html_content)

def get_manage_member_content(user):
    if settings.DEBUG:
        site_url = ''
    else:
        site_url = "https://" + Site.objects.get_current().domain
    context = {
        'today': timezone.localtime(timezone.now()),
        'user': user,
        'site_url': site_url,
    }
    return render_templates(context, "manage_member")

# Deprecated and moved to clean_mailgun_data
# TODO - remove
# def clean_incoming(request):
#     if not request.POST:
#         raise MailgunException("Request not a POST!")
#
#     header_txt = request.POST.get('message-headers')
#     message_headers = json.loads(header_txt)
#     message_header_keys = [item[0] for item in message_headers]
#
#     # A List-Id header will only be present if it has been added manually in
#     # this function, ie, if we have already processed this message.
#     if request.POST.get('List-Id') or 'List-Id' in message_header_keys:
#         raise MailgunException("List-Id header was found!")
#
#     # If 'Auto-Submitted' in message_headers or message_headers['Auto-Submitted'] != 'no':
#     if 'Auto-Submitted' in message_header_keys:
#         raise MailgunException("Message appears to be auto-submitted")
#
#     # Pull the variables out of the POST
#     recipient = request.POST.get('recipient')
#     from_address = request.POST.get('from')
#     subject = request.POST.get('subject')
#     body_plain = request.POST.get('body-plain')
#     body_html = request.POST.get('body-html')
#     logger.debug("Incoming from: %s, to: %s, subject: %s" % (from_address, recipient, subject))
#
#     # Add in a footer
#     text_footer = "\n\n-------------------------------------------\n*~*~*~* Sent through Nadine *~*~*~* "
#     body_plain = body_plain + text_footer
#     if body_html:
#         html_footer = "

-------------------------------------------
*~*~*~* Sent through Nadine *~*~*~* "
#         body_html = body_html + html_footer
#
#     # Build and return our data
#     mailgun_data = {"from": from_address,
#                     "to": [recipient, ],
#                     "subject": subject,
#                     "text": body_plain,
#                     "html": body_html,
#                     }
#
#     attachments = []
#     for attachment in request.FILES.values():
#         attachments.append(("inline", attachment))
#
#     return (mailgun_data, attachments)
#
#
# Deprecated and moved to comlink
# TODO - remove all 3
# @csrf_exempt
# def staff(request):
#     try:
#         mailgun_data, attachments = clean_incoming(request)
#     except MailgunException as e:
#         # mailgun requires a code 200 or it will continue to retry delivery
#         return HttpResponse(status=200)
#
#     # Pull the list this should go to and set them in the BCC
#     # Going out to all Users that are active and staff
#     bcc_list = list(User.objects.filter(is_staff=True, is_active=True).values_list('email', flat=True))
#     mailgun_data["bcc"] = bcc_list
#
#     # This goes out to staff and anyone that isn't already in the BCC list
#     to_list = ["staff@%s" % settings.MAILGUN_DOMAIN, ]
#     for to in mailgun_data["to"]:
#         if not to in bcc_list:
#             to_list.append(to)
#     mailgun_data["to"] = to_list
#
#     # Send the message
#     return mailgun_send(mailgun_data, attachments)
#
# @csrf_exempt
# def team(request):
#     try:
#         mailgun_data, attachments = clean_incoming(request)
#     except MailgunException as e:
#         # mailgun requires a code 200 or it will continue to retry delivery
#         return HttpResponse(status=200)
#
#     # Goes out to all managers
#     bcc_list = list(User.helper.managers(include_future=True).values_list('email', flat=True))
#     mailgun_data["bcc"] = bcc_list
#
#     # Hard code the recipient to be this address
#     to_list = ["team@%s" % settings.MAILGUN_DOMAIN, ]
#     for to in mailgun_data["to"]:
#         if not to in bcc_list:
#             to_list.append(to)
#     mailgun_data["to"] = to_list
#
#     # Send the message
#     return mailgun_send(mailgun_data, attachments)
#
# @csrf_exempt
# def test(request):
#     print("Request: ")
#     print(request)
#
#     try:
#         mailgun_data, attachments = clean_incoming(request)
#     except MailgunException as e:
#         # mailgun requires a code 200 or it will continue to retry delivery
#         return HttpResponse(status=200)
#
#     mailgun_data["to"] = ["test80085@%s" % settings.MAILGUN_DOMAIN, ]
#     mailgun_data["bcc"] = ['jsayles@gmail.com']
#     #return mailgun_send(mailgun_data, attachments)
#     return HttpResponse(status=200)


# csgo item crawler
# needs python 3.3
# install BeautifulSoup http://www.crummy.com/software/BeautifulSoup/
# download and install Tor for anonymous scrape/avoid 429 https://www.torproject.org/

import random
import time
import urllib.request
from bs4 import BeautifulSoup

random.seed()

minDelay = 1.0
delayTime = 2.0
pageId = 1
url = 'http://steamcommunity.com/market/search?q=&category_730_ItemSet%5B%5D=any&category_730_TournamentTeam%5B%5D=any&category_730_Weapon%5B%5D=any&category_730_Rarity%5B%5D=tag_Rarity_Rare_Weapon&category_730_Rarity%5B%5D=tag_Rarity_Uncommon_Weapon&category_730_Rarity%5B%5D=tag_Rarity_Mythical_Weapon&category_730_Rarity%5B%5D=tag_Rarity_Legendary_Weapon&category_730_Rarity%5B%5D=tag_Rarity_Ancient_Weapon&category_730_Rarity%5B%5D=tag_Rarity_Rare&category_730_Rarity%5B%5D=tag_Rarity_Mythical&category_730_Rarity%5B%5D=tag_Rarity_Legendary&category_730_Rarity%5B%5D=tag_Rarity_Contraband&category_730_Type%5B%5D=tag_CSGO_Type_Pistol&category_730_Type%5B%5D=tag_CSGO_Type_SMG&category_730_Type%5B%5D=tag_CSGO_Type_Rifle&category_730_Type%5B%5D=tag_CSGO_Type_Shotgun&category_730_Type%5B%5D=tag_CSGO_Type_SniperRifle&category_730_Type%5B%5D=tag_CSGO_Type_Machinegun&category_730_Type%5B%5D=tag_CSGO_Type_Knife&category_730_Type%5B%5D=tag_CSGO_Tool_WeaponCase_KeyTag&appid=730'
itemList = []
index = 0

for i in range(1):
    #try:
    fh = urllib.request.urlopen(url + '#p' + str(pageId) + '_popular_desc')
    soup = BeautifulSoup(fh)

    items = soup.find_all("span", attrs={"class":"market_listing_item_name"})
    for item in items:
        itemList.append(dict([('name', item.contents)]))

    price = 1
    # price then quantity
    listings = soup.find_all("span", attrs={"class":"market_table_value"})
    for listing in listings:
        if price == 1:
            itemList[index]['price']=listing.span.contents
            price = 0
        else:
            itemList[index]['quantity']=listing.span.contents
            price = 1
            index += 1

    #finally:
    fh.close()
    pageId += 1
    print('page ' + str(i))
    time.sleep(minDelay + random.random()*delayTime)

totalValue = 0.0
totalItems = 0;
for item in itemList:
    print(str(item['name'][0]) + ',' + str(item['price'][0]) + ',' + str(item['quantity'][0]))
    totalValue += float(str(item['price'][0]).strip('$')) * float(str(item['quantity'][0]).replace(',',''))
    totalItems += int(str(item['quantity'][0]).replace(',',''))
print('totalValue = $' + str(totalValue))
print('totalItems = ' + str(totalItems))


import sys
import numpy as np
from scipy import sparse
from time import time

def sim(u1, u2, sig):
	"""
	Measure and return the similarity between the signatures (sig) of two users (u1, u2).
	"""
	similarity = np.sum(np.equal(sig[:,u1],sig[:,u2])).astype(float)/len(sig[:,u1])
	return similarity

def jsim(u1, u2, sm):
    """Calculate and return the Jaccard similarity between two users
    (u1 and u2) with the sparse matrix sm """
    a = set(sm.indices[sm.indptr[u1]:sm.indptr[u1+1]])
    b = set(sm.indices[sm.indptr[u2]:sm.indptr[u2+1]])
    c = a.intersection(b)
    return float(len(c)) / (len(a) + len(b) - len(c))

def band_to_bucket(band):
	"""
	map each column in a band to buckets as a dictionary,
	return the list of items within the same bucket
	"""
	buckets={}
	for c in range(band.shape[1]):
		buckets.setdefault(tuple(band[:,c]),[]).append(c)
	return [locs for key,locs in buckets.items() if len(locs)>1]

def main():
	before=time()
	#read 2 arguments: random seed and path of the data
	seeds = int(sys.argv[1])
	datafile = sys.argv[2]

	print("Loading the data......")
	data = np.load(datafile)

	#construct sparse matrix of (N_user, N_movie)
	N_usr = np.max(data[:,0])+1
	N_mov = np.max(data[:,1])+1
	V = np.ones(data.shape[0])
	A = sparse.csc_matrix((V,(data[:,1],data[:,0])),shape=(N_mov,N_usr))

	#size of signature
	N_sig = 100
	print("Generating signature......")
	sig = np.zeros((N_sig,N_usr)).astype('int32')
	for k in range(N_sig):
		#generate random permutation
		np.random.seed(seeds+k)
		A_permute = np.random.permutation(N_mov)
		#apply permutation to the sparse matrix
		A_new = A[A_permute,:]
		#find the index of the first non-zero element in each column
		for j in range(N_usr):
			ar = A_new.indices[A_new.indptr[j]:A_new.indptr[j + 1]].min()
			sig[k,j] = ar

	print("Signature is generated.")

	b=20
	r=5

	#split the signature matrix into b=20 bands with r=5 rows each.
	subarrays = np.split(sig, range(r, N_sig, r))

	print("Generating candidates......")
	#'sim_list' containing the priliminary similar pairs by comparing the signature similarity
	sim_list=[]
	for band in subarrays:
		pairs = band_to_bucket(band.astype(int))
		for bucket in pairs:
			#ignore the buckets with size larger than a threshold 200.
			if len(bucket)<200:
				sim_list+=[(u1,u2) for u1 in bucket for u2 in bucket \
				if u1<u2 if sim(u1,u2,sig)>0.5]
	sim_list=set(sim_list)

	print("Sifting the candidates to find the true similar pairs......")
	#'sim_list' containing the true similar pairs by comparing the jaccard similarity of full data
	user_pair_list=[]
	for pair in sim_list:
	    # if jaccard similarity > 0.5, add to the list
	    similarity = jsim(pair[0], pair[1], A)
	    if similarity > 0.5:
	        user_pair_list.append((pair[0], pair[1], similarity))

	N_pairs = len(user_pair_list)
	print("Number of pairs found: {0}".format(N_pairs))
	print("Number of pairs per minute: {0:.1f}".format(N_pairs/((time()-before)/60.0)))
	user_pair_list.sort()
	outfile = 'user-pairs.txt'
	print("Saving to file......")
	np.savetxt(outfile, user_pair_list, fmt="%d,%d,%.6f", delimiter=',')

if __name__ == '__main__':
	main()


"""Program used to test the implementation of our classes"""

import farm
from farm import Farm
import sensor
from sensor import Sensor
import score

def main():
    testfarm = Farm((0,0))
    senone = Sensor((44.790300, -92.931998))
    sentwo = Sensor((44.790300, -92.931998))
    senthree = Sensor((44.790300, -92.931998))

    testfarm.add_sensor(senone)
    testfarm.add_sensor(sentwo)
    testfarm.add_sensor(senthree)

    testfarm.pull_current()
    testfarm.pull_forecast()

    print(testfarm.future_forecast)

    #prints out some random weather data from sensors
    """
    i = 0
    while i < len(testfarm.sensors):
        print(testfarm.sensors[i].point)
        i += 1
    """

    returnlist = score.ranker(testfarm, testfarm.sensors[1])

    print(returnlist)

main()


import os, requests, sys
from pathlib import Path
from dotenv import load_dotenv
from utils import print_center, print_loop

# .env reader
env_file = '.env'
env_path = Path('.') / env_file
load_dotenv(dotenv_path=env_path)

# Colors
light_green = "\033[92m"
yellow = "\033[33m"
bold = "\033[1m"
blue = '\033[34m'
red = '\033[31m'
eoc = "\033[0m"

# OpenWeatherMap
open_weather_map_api_key = os.getenv('OPEN_WEATHER_MAP_API_KEY')
open_weather_map_version = os.getenv('OPEN_WEATHER_MAP_VERSION') or '2.5'
city_name = os.getenv('OPEN_WEATHER_MAP_CITY') or 'Montreuil'
country_code = os.getenv('OPEN_WEATHER_MAP_COUNTRY_CODE') or 'fr'
base_url = "http://api.openweathermap.org/data/" + open_weather_map_version + "/weather?"
location = city_name + "," + country_code
units = "metric"
complete_url = base_url + "appid=" + open_weather_map_api_key + "&q=" + location + "&units=" + units

def get_weather():
  res = requests.get(complete_url).json()
  if res['cod'] == 200:
    main = res['main']
    weather = res['weather'][0]
    temp = str(main['temp']) + 'ºC'
    temp_max = main['temp_max']
    temp_min = main['temp_min']
    max_min = str(temp_min) + ' / ' + str(temp_max)
    description = weather['description']
    location_name = 'Weather at ' + bold + city_name + eoc
    return temp, max_min, description, location_name
  elif res['cod'] == '404':
    print('City not found');
    exit()
  else:
    print('An error occured')
    exit()

def print_weather():
  temp, max_min, description, location_name = get_weather()
  print(blue)
  print_center(location_name, 16)
  print(blue)
  print_center(temp)
  print_center(max_min)
  print_center(description)
  print(eoc)

if sys.argv[0] == __file__:
  print_loop(print_weather)


#calss header
class _HOTHEAD():
	def __init__(self,):
		self.name = "HOTHEAD"
		self.definitions = [u'someone who does things or reacts to things quickly and without thinking carefully first']

		self.parents = []
		self.childen = []
		self.properties = []
		self.jsondata = {}

		self.specie = 'nouns'

	def run(self, obj1 = [], obj2 = []):
		return self.jsondata


def mergeSort(arr, start, end):
    if start < end:
        middle = (start + end) // 2
        mergeSort(arr, start, middle)
        mergeSort(arr, middle + 1, end)
        merge(arr, start, middle, end)

# 합치는 함수
def merge(arr, start, middle, end):
    new = []
    s = start
    m = middle + 1
    while s <= middle and m <= end:
        if arr[s] < arr[m]:
            print(start, middle, end, new)
            new.append(arr[s])
            s += 1
        else:
            new.append(arr[m])
            m += 1
    while s <= middle:
        new.append(arr[s])
        s += 1
    while m <= end:
        new.append(arr[m])
        m += 1
    print(start, middle, end, new)
    for i in range(len(new)):
        arr[start] = new[i]
        start += 1

arr = [12, 31, 24, 14, 24, 1, 4, 125, 1, 51, 25, 313, 1, 5, 1, 7, 71, 7, 17, 37, 274273]
# arr = [1, 23, 41, 5, 123, 12, 14, 3, 412, 1, 41241, 2, 35, 57, 676, 75, 7670]
print("before:", arr)
new = [0] * int(len(arr) - 1)
mergeSort(arr, 0, len(arr) - 1)
print("after:", arr)


from DataParser import DataParser
from functions import linear_regression, principal_component_analysis
import pandas as pd
import numpy as np
from queue import Queue

def extract(data: pd.DataFrame):
    y = data["OMX"]
    x = np.array(data["BCOMPR"]).reshape((-1, 1))
    return x, y

def print_model(x: np.array, y: np.array):
    model = linear_regression(x, y)
    b0 = model.intercept_
    b1 = model.coef_
    print("The model is:")
    print("yt = %f + %s + σ*εt\n" % (b0, " + ".join(["(%f * x%dt)" % (v, i + 1) for (i, v) in enumerate(b1)])))
    print("Score:", model.score(x, y), "\n")

def moving_average(data: np.array, window: int):
    q = Queue()
    current_total = 0
    averages = []
    for i in range(window - 1):
        current_total += data[i]
        q.put(data[i])
        averages.append(pd.NaT)
    for i in range(window - 1, len(data)):
        current_total += data[i]
        averages.append(current_total / window)
        current_total -= q.get()
        q.put(data[i])
    return averages

def set_moving_average(data: pd.DataFrame, *args):
    for key in data.keys():
        for window in args:
            data["%s_ma_%d" % (key, window)] = moving_average(np.array(data[key][0:]), window)
    data = data.dropna(axis=0)
    return data

def main():
    file = "data/data.json"
    full_file = "data/full_dataset.json"
    DataParser.load(file)
    DataParser.load_full(full_file)
    data = DataParser.get()
    x, y = extract(data)
    print("Question 1:\n")
    print_model(x, y)
    # full_dataset = set_moving_average(DataParser.full_dataset(), 20, 50, 200)
    print("===========================================================")
    print("Question 2:\n")
    principal_component_analysis(DataParser.full_dataset(), y)

if __name__ == "__main__":
    main()


    #!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Sep  3 11:25:14 2020

@author: nooteboom
"""
import os
assert os.environ['CONDA_DEFAULT_ENV']=='Cartopy-py3', 'You should use the Cartopy_py3 conda environment here'
import numpy as np
import network_functions as nwf
from sklearn.metrics.pairwise import euclidean_distances
from netCDF4 import Dataset

#%% vars
sp = 250 # sinking speed (m/day)
maxlat = 65 # use the sedimentary dat below maxlat deg N
for mins in [50,100, 200, 300, 400, 500, 600,700,800,900, 1000]: # s_min values
    #%% Load the species data
    # Read Foram data
    readData = '/Volumes/HD/network_clustering/'
    data = nwf.readForamset(readData + 'ForamData.csv')
    Foramspecies = nwf.readForamset(readData + 'ForamDataHeader.txt')[0][21:]
    data[data=='N/A'] = '0'#'NaN'
    data = data[:, 21:].astype(np.float)
    Flats, Flons = nwf.readForamset_lonlats(readData + 'ForamData.csv')
    Flons[np.where(Flons<0)]+=360

    # Read Dino data
    FlonsDino, FlatsDino, Dinodata = nwf.readDinoset(readData+'dinodata_red.csv')
    FlonsDino[FlonsDino<0] += 360
    Dinodata[np.isnan(Dinodata)] = 0

    # limit the data below maxlat
    idx = np.where(Flats<maxlat)[0]
    Flons = Flons[idx]
    Flats = Flats[idx]
    data = data[idx]
    idx = np.where(FlatsDino<maxlat)[0]
    FlonsDino = FlonsDino[idx]
    FlatsDino = FlatsDino[idx]
    Dinodata = Dinodata[idx]

    #%% The dinstance matrix based on taxonomy

    Ftaxdist = nwf.euclidean_distances(data)
    Dinotaxdist = nwf.euclidean_distances(Dinodata)
    Ftaxdist = nwf.normalise(Ftaxdist)
    Dinotaxdist = nwf.normalise(Dinotaxdist)

    #%% The distance matrix based on the distance betweeen sites

    Fsitedist = nwf.direct_embedding(Flons, Flats)
    Fsitedist = nwf.euclidean_distances(Fsitedist)
    Dinositedist = nwf.direct_embedding(FlonsDino, FlatsDino)
    Dinositedist = nwf.euclidean_distances(Dinositedist)

    Fsitedist = np.round(nwf.normalise(Fsitedist),4)
    Dinositedist = np.round(nwf.normalise(Dinositedist),4)

    assert nwf.check_symmetric(Fsitedist, 0, 0)
    assert nwf.check_symmetric(Dinositedist, 0, 0)
    #%% The distance matrix based on the environmental variables
    from scipy.interpolate import griddata

    dirRead = '/Users/nooteboom/Documents/GitHub/cluster_TM/cluster_SP/density/dens/matrices/WOAdata/'
    envs = ['temp', 'salt', 'P', 'N', 'Si']
    files = {
            'temp':'woa18_95A4_t00_01.nc',
            'salt':'woa18_95A4_s00_01.nc',
            'P':'woa18_all_p00_01.nc',
            'N':'woa18_all_n00_01.nc',
            'Si':'woa18_all_i00_01.nc',
            }
    varss = {
            'temp':'t_an',
            'salt':'s_an',
            'P':'p_an',
            'N':'n_an',
            'Si':'i_an',
            }
    Fenv = np.zeros((len(Flons), len(envs)))
    Dinoenv = np.zeros((len(FlonsDino), len(envs)))

    for eni, en in enumerate(envs):
        ncf = Dataset(dirRead + files[en])
        var = ncf[varss[en]][0,0].flatten()
        lon = ncf['lon'][:]
        lon[lon<0] += 360
        lat = ncf['lat'][:]
        lat, lon = np.meshgrid(lat, lon)
        lon = lon.flatten(); lat = lat.flatten()
        idx = (~var.mask)
        var = var[idx]; lon = lon[idx]; lat = lat[idx];

        points = np.concatenate((lat[:,np.newaxis],lon[:,np.newaxis]), axis=1)

        res = griddata(points, var, (Flats, Flons), method='nearest')
        res = nwf.normalise2(res)
        Fenv[:,eni] = res
        res = griddata(points, var, (FlatsDino, FlonsDino), method='nearest')
        res = nwf.normalise2(res)
        Dinoenv[:,eni] = res

    Dinoenvdist = euclidean_distances(Dinoenv)
    Fenvdist = euclidean_distances(Fenv)
    assert nwf.check_symmetric(Fenvdist)
    assert nwf.check_symmetric(Dinoenvdist)
    #%% The distance matrix based on the OPTICS clustering

    dirr = '/Users/nooteboom/Documents/GitHub/cluster_TM/cluster_SP/density/dens/results/'
    ff = np.load(dirr+'OPTICS_sp%d_smin%d.npz'%(sp, mins))
    lon0 = ff['lon']
    lat0 = ff['lat']
    reach = ff['reachability']
    ordering = ff['ordering']
    predecessor = ff['predecessor']
    core_distances = ff['core_distances']

    lon0 = lon0[ordering][1:]
    lat0 = lat0[ordering][1:]
    reach = reach[ordering][1:]

    args = nwf.find_nearest_args(lon0, lat0, Flats, Flons)
    Freachdist = nwf.dis_from_reach(reach, args)
    Freachdist = nwf.normalise(Freachdist)

    args = nwf.find_nearest_args(lon0, lat0, FlatsDino, FlonsDino)
    Dinoreachdist = nwf.dis_from_reach(reach, args)
    Dinoreachdist = nwf.normalise(Dinoreachdist)

    assert nwf.check_symmetric(Freachdist)
    assert nwf.check_symmetric(Dinoreachdist)

    #%% Save distance matrices

    np.savez('dms/distance_matrices_sp%d_smin%d'%(sp, mins),
             Fsitedist=Fsitedist, Ftaxdist=Ftaxdist, Freachdist=Freachdist, Fenvdist=Fenvdist,
             Dinositedist=Dinositedist, Dinotaxdist=Dinotaxdist, Dinoreachdist=Dinoreachdist,
             Dinoenvdist=Dinoenvdist
             )


import cv2

def create_sift_features(img):

	sift = cv2.xfeatures2d.SIFT_create()  # create SIFT object

	# Detect keypoints:
	# kp: keypoints, desc: descriptor

	kp, desc = sift.detectAndCompute(img, None)

	# show keypoints:
	img_draw_kp = cv2.drawKeypoints(img, kp, img)

	return kp, desc, img_draw_kp


import unittest
import random
from core.avl import AVL

class AVLTester(unittest.TestCase):
    def setUp(self):
        self.avl = AVL()
        #insert random permutation of [1, ..., 1000] list to avl tree
        ls = list(range(1, 1000))
        random.shuffle(ls)
        for v, k in enumerate(ls):
            self.avl.insert(k, v + 1)

    def test_is_all(self):
        test_set = set(range(1, 1000))
        #get value by key works fine
        all_values = {self.avl.get_value_by_key(x) for x in range(1, 1000)}
        self.assertSetEqual(all_values, test_set, "get_value_by_key fails")
        #get value by position works fine
        all_values = {self.avl.get_value_by_position(x) for x in range(1, 1000)}
        self.assertSetEqual(all_values, test_set, "get_value_by_position fails")
        #tree iterator works fine
        all_values = {x.value for x in self.avl}
        self.assertSetEqual(all_values, test_set, "tree iterator fails")

    def test_special_cases(self):
        self.assertIsNone(self.avl.get_value_by_key(-7))
        self.assertIsNone(self.avl.get_value_by_key(6666))
        self.assertIsNone(self.avl.get_value_by_position(6666))
        self.assertEqual(self.avl.get_value_by_position(-21), self.avl.get_value_by_position(1))

class AVLTester2(unittest.TestCase):
    def setUp(self):
        self.avl = AVL()

    def test_some_things(self):
        self.avl.insert(277, 19) #some random values choosen by me
        self.avl.insert(31, 14) #order of inserted elements
        self.avl.insert(91, 99) #forces rotation of nodes
        self.avl.insert(99, 22)
        self.avl.insert(111, 78)
        self.avl.insert(121, 91)
        self.assertListEqual([78, 91, 19], [x.value for x in self.avl.get_iterator_by_key(111)])
        self.assertListEqual([78, 91, 19], [x.value for x in self.avl.get_iterator_by_position(4)])

suite1 = unittest.TestLoader().loadTestsFromTestCase(AVLTester)
suite2 = unittest.TestLoader().loadTestsFromTestCase(AVLTester2)
all_tests = unittest.TestSuite([suite1, suite2])
unittest.TextTestRunner(verbosity=2).run(all_tests)


"""HTML: PLAIN Writer Style

The plain style writes the document as is. That is, the data in the
text node is printed without any processing.

Processing instruction gets written as

    <?target*

where `*` is a whitespace if the content has no newline. Otherwise
`*` is the newline character. Also, since the attributes do not
collect any of the spaces or newline characters, the plain style
cannot rewrite a file in the same way it was parsed. That is, if the
file was parsed from

    <parent att1="val1"
            att2="val2">
        <child></child>
    </parent>

then this style will print

    <parent att1="val1" att2="val2">
        <child></child>
    </parent>

"""

from lexor import init, load_aux

INFO = init(
    version=(0, 0, 1, 'final', 0),
    lang='html',
    type='writer',
    description='Writes HTML files with no text node processing.',
    url='http://jmlopez-rod.github.io/lexor-lang/html-writer-plain',
    author='Manuel Lopez',
    author_email='jmlopez.rod@gmail.com',
    license='BSD License',
    path=__file__
)
MOD = load_aux(INFO)['nw']
MAPPING = {
    '#comment': MOD.CommentNW,
    '#doctype': MOD.DoctypeNW,
    '#cdata-section': MOD.CDataNW,
    '__default__': MOD.DefaultNW,
}


import tkinter as tk
import main

bgCol = "#4B064B"

# Main Window
win = tk.Tk()
win.geometry('400x180')
win.title("Covid Tracker")
win.configure(bg=bgCol)

# Header
headerFrame = tk.Frame(win)
headerFrame.pack(fill=tk.X)

headText = tk.Label(headerFrame)
headText.configure(text='COVID TRACKER')
headText.pack()

# Input
inpFrame = tk.Frame(win)
inpFrame.configure(bg=bgCol)
inpFrame.pack(pady=20,fill=tk.X)

inpEntry = tk.Entry(inpFrame)
inpEntry.configure(width=40)
inpEntry.pack(pady=10)

stateHead = tk.Label(win,text='')

def onClick():

    inpFrame.pack(side='bottom')

    win.geometry('800x800')

    userInput = inpEntry.get()
    stateData = main.mainCall(userInput)

    stateHead.configure(text=stateData['state'])
    stateHead.configure(font=('Helvetica',20))
    stateHead.place(x=20,y=20)

    i = 0
    for key in stateData:

        keyLabel = tk.Label(win,text=key.upper(),bg=bgCol,fg='white')
        keyLabel.place(x=20,y=100 + i)

        valLabel = tk.Label(win,text=stateData[key],bg=bgCol,fg='white')
        valLabel.place(x=200,y=100 + i)

        i = i + 40

#Button
trackButton = tk.Button(inpFrame,text="Track",width=20)
trackButton.pack(pady=20,side='bottom')
trackButton.configure(command=onClick)

win.mainloop()


import math
from State import State
import numpy as np

class World(object):
    """description of class"""
    def __init__(self, arg):
        self.width = arg[0]+1
        self.height = arg[1]+1

        self.map = np.zeros([self.width, self.height])
        self.obstacle = 1
        self.free = 0

    def getPathCosts( self, args):
        tasks = args
        for task in tasks:
            for task2 in tasks:
                [path, length] = self.aStar(task, task2)

    def aStar(self, arg):
        start = State([arg[0].x, arg[0].y]) # state
        goal = State([arg[1].x, arg[1].y]) # state
        epsilon = arg[2]

        start.g = 0
        start.f = epsilon * self.heuristic( [start, goal] )

        o_set = []
        c_set = []
        o_set.append( start )

        while len(o_set) > 0:
            c = self.getMindex(o_set)
            current = o_set[c]
            o_set.remove(current)
            c_set.append( current )

            if abs(current.x - goal.x) +  abs(current.y - goal.y) < 1:
                path = []
                length = 0
                path.append(current)
                while abs(current.x - start.x) +  abs(current.y - start.y) > 0.1:
                    prev = current.cameFrom
                    path.append( prev )
                    length += self.heuristic([prev, current])
                    current = prev

                path.reverse()
                if length > 0:
                    path.pop(0)

                return [path, length]

            nbrs = self.getNbrs( current )

            for nbr in nbrs:
                if self.map[nbr.x, nbr.y] == self.free and not self.inList([c_set, nbr]):
                    if not self.inList([o_set, nbr]):
                        nbr.g = current.g + 1
                        nbr.f = nbr.g + epsilon * self.heuristic([nbr, goal])
                        nbr.cameFrom = current
                        o_set.append( nbr )
                    elif nbr.g > current.g + 1:
                        nbr.g = current.g + 1
                        nbr.f = nbr.g + epsilon * self.heuristic([nbr, goal])
                        nbr.cameFrom = current

        return [-1, float("inf")]

    def inList(self, arg):
        list = arg[0]
        item = arg[1]

        for s in list:
            if s.x == item.x and s.y == item.y:
                return True

        return False

    def getNbrs(self, arg):
        c = State([arg.x, arg.y])
        nbrs = []

        nx = [-1,-1,-1, 0,0, 1,1,1]
        ny = [-1, 0, 1,-1,1,-1,0,1]

        for x,y in zip(nx, ny):
            n = State([c.x+x, c.y+y])
            if n.x > -1 and n.x < self.width and n.y > -1 and n.y < self.height and self.map[n.x, n.y] == self.free:
                nbrs.append( n )

        return nbrs

    def getMindex(self, arg):
        set = arg
        min = float("inf")
        mindex = -1
        for i, s in enumerate(set):
            if s.f < min:
                min = s.f
                mindex = i

        return mindex

    def heuristic(self, arg):
        return math.sqrt(math.pow(arg[0].x - arg[1].x,2) + pow(arg[0].y - arg[1].y,2))