#!/bin/python

from urllib2 import urlopen
from collections import namedtuple
from subprocess import check_call, Popen, PIPE
import socket
import fcntl
import struct
import platform
import getpass
import os

# Written by: Phantom Raspberry Blower (The PRB)
# Date: 01-10-2018
# Description: Script for displaying system information
#              about software, CPU, hardware, memory,
#              storage, networks and addons installed.
#              Tested on Raspberry Pi.

class SystemInfo():

    # ## Software Information ## #

    @property
    def username(self):
        # Get username
        return getpass.getuser()

    @property
    def hostname(self):
        # Get hostname
        return socket.gethostname()

    @property
    def is_user_root(self):
        # Is current user root
        if os.geteuid() == 0:
            return True
        else:
            return False

    @property
    def os_platform(self):
        return platform.platform()

    @property
    def platform_system(self):
        return platform.system()

    @property
    def platform_node(self):
        return platform.node()

    @property
    def platform_release(self):
        return platform.release()

    @property
    def platform_version(self):
        return platform.version()

    @property
    def platform_machine(self):
        return platform.machine()

    @property
    def platform_processor(self):
        return platform.processor()

    @property
    def platform_architecture(self):
        return platform.architecture()

    @property
    def platform_python_build(self):
        return platform.python_build()

    @property
    def platform_python_compiler(self):
        return platform.python_compiler()

    @property
    def platform_python_branch(self):
        return platform.python_branch()

    @property
    def platform_python_implementation(self):
        return platform.python_implementation()

    @property
    def platform_python_revision(self):
        return platform.python_revision()

    @property
    def platform_python_version(self):
        return platform.python_version()

    @property
    def platform_python_version_tuple(self):
        return platform.python_version_tuple()

    @property
    def platform_win32_ver(self):
        # Windows platforms
        return platform.win32_ver()

    @property
    def platform_mac_ver(self):
        return platform.mac_ver()

    @property
    def platform_linux_distribution(self):
        # Unix platforms
        return platform.linux_distribution()

    # ## CPU Information ## #

    @property
    def cpu_model(self):
        # Get Model Name
        return self._get_cpu_item('model name', '')

    @property
    def cpu_hardware(self):
        # Get Hardware
        return self._get_cpu_item('Hardware', '')

    @property
    def cpu_revision(self):
        # Get Revision
        return self._get_cpu_item('Revision', '')

    @property
    def cpu_serial(self):
        # Get CPU Serial Number
        return self._get_cpu_item('Serial', '0000000000000000')

    @property
    def cpu_cores(self):
        # Get number of cpu cores
        # Linux, Unix and MacOS:
        if hasattr(os, "sysconf"):
            if "SC_NPROCESSORS_ONLN" in os.sysconf_names:
                # Linux and Unix:
                ncpus = os.sysconf("SC_NPROCESSORS_ONLN")
                if isinstance(ncpus, int) and ncpus > 0:
                    return ncpus
            else: # OSX:
                return int(os.popen2("sysctl -n hw.ncpu")[1].read())

    @property
    def cpu_temp(self):
        # Return CPU temperature as a character string
        try:
            output = Popen(['vcgencmd', 'measure_temp'], stdout=PIPE).communicate()[0]
            return str(output[5:len(output)-3])
        except:
            return 'Unable to get CPU temperature! :('

    # ## Memory Informtion ## #

    # Get memory info
    @property
    def ram_info(self):
        _ntuple_RAMinfo = namedtuple('RAM', 'total used free')
        output = Popen(['free'], stdout=PIPE).communicate()[0]
        for line in output.split('\n'):
            if 'Mem:' in line:
                total = line.split()[1]
                used = line.split()[2]
                free = int(total) - int(used)
        return _ntuple_RAMinfo(total, used, free)

    # ## Storage Informtion ## #

    # Get hard drive(s) info
    @property
    def disk_info(self):
        devices = []
        devices.append(self.disk_usage("/root/"))
        for device in self.get_devices("/media/"):
            devices.append(self.disk_usage("/media/" + device + "/"))
        return devices

    # ## Network Informtion ## #

    @property
    def wan_ip_addr(self):
        # Get the WAN IP address
        try:
            return urlopen('http://ip.42.pl/raw').read()
        except:
            return 'Unable to get WAN IP Address! :('

    @property
    def default_gateway(self):
        # Get default gateway
        with open("/proc/net/route") as fh:
            for line in fh:
                fields = line.strip().split()
                if fields[1] != '00000000' or not int(fields[3], 16) and 2:
                    continue
                return socket.inet_ntoa(struct.pack("<L", int(fields[2], 16)))

    @property
    def arp_info(self):
        # Discover other devices on the network
        import re
        devices = []
        s = Popen(["arp"], stdout=PIPE).communicate()[0]
        for item in s.split('\n'):
            devices.append("%s" % item.strip())
        return devices

    ## # Functions # ##

    # ## Software Information ## #

    def get_platform_info(self):
        distname, dist_version, id = platform.linux_distribution()
        system, node, release, version, machine, processor = platform.uname()
        return (self.os_platform,
                distname,
                dist_version,
                id,
                system,
                node,
                release,
                version,
                machine,
                processor)

    def get_installed_addons(self, dir=None):
        if not dir == None:
            output = [dI for dI in os.listdir(dir) if os.path.isdir(os.path.join(dir, dI))]
            output.sort()
            return output

    # ## CPU Information ## #

    # Get CPU items
    def _get_cpu_item(self, item, ini_value):
        temp = ini_value
        try:
            f = open('/proc/cpuinfo', 'r')
            for line in f:
                if line[0:len(item)] == item:
                    item_value = line[len(item)+3:]
            f.close()
        except:
            item_value = 'Unable to get %s! :(' % item
        return item_value.strip()

    # ## Hard Disk Informtion ## #

    # Get Hard Drive Devices
    def get_devices(self, path):
        return next(os.walk(path))[1]

    # Get hard drive usage
    def disk_usage(self, path):
        _ntuple_diskusage = namedtuple('usage', 'path total used free')
        st = os.statvfs(path)
        free = st.f_bavail * st.f_frsize
        total = st.f_blocks * st.f_frsize
        used = (st.f_blocks - st.f_bfree) * st.f_frsize
        return _ntuple_diskusage(path, total, used, free)

    # ## Network Informtion ## #

    # Get list of network settings
    def get_network_settings(self):
        _ntuple_LANinfo = namedtuple('LAN', 'ssid psk proto')
        ssid = ''
        psk = ''
        proto = ''
        settings = []
        if os.path.exists('/var/lib/connman'):
            for network in next(os.walk('/var/lib/connman'))[1]:
                try:
                    settings.append((network, ''))
                    file = open('/var/lib/connman/%s/settings' % network, 'r')
                    for line in file:
                        settings.append((line[:line.find('=')],
                                         line[line.find('=')+1:].replace('\n',
                                         '')))
                    file.close()
                    settings.append(('', ''))
                except:
                    pass
        elif os.path.exists('/etc/wpa_supplicant/wpa_supplicant.conf'):
            file = open('/etc/wpa_supplicant/wpa_supplicant.conf', 'r')
            for line in file:
                settings.append((line[:line.find('=')].strip(),
                                 line[line.find('=')+1:].replace('\n',
                                 '')))
            file.close()
        else:
            return 'Unable to get Network Settings! :('
        return settings

    # Get list of networks
    def get_networks_info(self):
        try:
            lst = []
            tabs = ''
            for name, value in self.get_network_settings():
                if name != value:
                    if value != '':
                        if len(name) < 10:
                            tabs = "\t\t\t"
                        elif len(name) < 18:
                            tabs = "\t\t"
                        elif len(name) < 24:
                            tabs = "\t"
                        else:
                            tabs = "\t\t"
                        lst.append(' %s:%s%s\n' % (name, tabs, value))
                else:
                    lst.append('%s\n' % name)
            return lst
        except:
            return 'Unable to get Networks! :('

    # Get LAN IP address for either etho or wlan
    def get_lan_ip_addr(self, ifname):
        try:
            s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
            return socket.inet_ntoa(fcntl.ioctl(
                                    s.fileno(),
                                    0x8915, # SIOCGIFADDR
                                    struct.pack('256s', ifname[:15])
                                    )[20:24])
        except:
            return 'Unable to get LAN IP address! :('

    # Get whois response
    def get_whois(self, ip):
        results = []
        if self.is_tool_installed('whois') is False:
            if self.is_user_root:
                self.install_tool('whois')
            else:
                return results
        desc = ['netname:', 'descr:', 'country:', 'role:']
        command = 'whois ' + ip
        output = Popen(command.split(), stdout=PIPE).communicate()[0]
        for line in output.split('\n'):
            for item in desc:
                if line[0:len(item)] == item:
                    results.append(line[len(item):].strip())
        return results

    # Check tool is installed
    def is_tool_installed(self, name):
        try:
            devnull = open(os.devnull)
            Popen([name],
                  stdout=devnull,
                  stderr=devnull).communicate()
        except OSError as e:
            if e.errno == os.errno.ENOENT:
                return False
        return True

    # Install Whois
    def install_tool(self, name):
        # Install tool and suppress output
        devnull = open(os.devnull, 'w')
        check_call(["sudo",
                    "apt-get",
                    "install",
                    "-y",
                    "-qq",
                    name], stdout=devnull, stderr=devnull)

    # Get complete system information
    def get_system_info(self):
        try:
            arp_info = self.arp_info
            disk_info = self.disk_info
            ram_info = self.ram_info
            networks_info = self.get_networks_info()
            whois_info = self.get_whois(self.wan_ip_addr)
            disk_info_txt = '\nStorage:'
            networks_info_txt = ''
            arp_info_txt = '\nNetwork Devices:\n'
            if self.is_user_root:
                networks_info_txt = '\nPrevious Networks:\n'
                for network in networks_info:
                    networks_info_txt = networks_info_txt + '\
                    %s' % (network)

            addons_info = self.get_installed_addons('/home/osmc/.kodi/addons')
            addons_info_txt = '\nAddons Installed:'
            for addon in addons_info:
                if (not addon == 'packages' and not addon == 'temp'
                    and not 'resource.language.' in addon):
                    addons_info_txt = addons_info_txt + '\n \
                    %s' % (addon)

            addons_info_txt += '\n'
            (os_platform,
             distname,
             dist_version,
             id,
             system,
             node,
             release,
             version,
             machine,
             processor) = self.get_platform_info()
            for item in disk_info:
                disk_info_txt = disk_info_txt + u'\n \
                    Path:\t\t\t%s\n \
                    Total:\t\t\t%sGB\n \
                    Used:\t\t\t%sGB\n \
                    Free:\t\t\t%sGB\n' % (
                    item.path,
                    item.total / (1024**3),
                    item.used / (1024**3),
                    item.free / (1024**3))

            for device in arp_info:
                arp_info_txt = arp_info_txt + '\t\t%s\n' % device

            return u'\nSoftware:\n \
                    Username:\t\t\t%s\n \
                    Hostname:\t\t\t%s\n \
                    Platform:\t\t\t%s\n \
                    Distribution:\t\t%s\n \
                    Dist. Version:\t\t%s\n \
                    System:\t\t\t%s\n \
                    Node:\t\t\t%s\n \
                    Release:\t\t\t%s\n \
                    Version:\t\t\t%s\n \
                    Machine:\t\t\t%s\n \
                    Processor:\t\t\t%s\n \
                    \nCPU:\n \
                    Model:\t\t\t%s\n \
                    Cores:\t\t\t%s\n \
                    Temp:\t\t\t%s\n \
                    Hardware:\t\t\t%s\n \
                    Revision:\t\t\t%s\n \
                    Serial number:\t\t%s\n \
                    \nMemory:\n \
                    Total:\t\t\t%sMB\n \
                    Used:\t\t\t%sMB\n \
                    Free:\t\t\t%sMB\n \
                    %s \
                    \nNetwork:\n \
                    LAN IP Address:\n \
                    \teth0:\t\t\t%s\n \
                    \twlan0:\t\t\t%s\n \
                    Gateway IP Address:\t%s\n \
                    WAN IP Address:\t\t%s\n \
                    Whois:\n \
                    \tnetname:\t\t%s\n \
                    \trole:\t\t\t%s\n \
                    \tdescr:\t\t\t%s\n \
                    \tcountry:\t\t%s\n \
                    %s \
                    %s \
                    %s' % (self.username,
                           self.hostname,
                           self.os_platform,
                           distname,
                           dist_version,
                           system,
                           node,
                           release,
                           version,
                           machine,
                           processor,
                           self.cpu_model,
                           self.cpu_cores,
                           self.cpu_temp,
                           self.cpu_hardware,
                           self.cpu_revision,
                           self.cpu_serial,
                           int(ram_info.total) / 1024,
                           int(ram_info.used) / 1024,
                           int(ram_info.free) / 1024,
                           disk_info_txt,
                           self.get_lan_ip_addr('eth0'),
                           self.get_lan_ip_addr('wlan0'),
                           self.default_gateway,
                           self.wan_ip_addr,
                           whois_info[0],
                           whois_info[3],
                           whois_info[1],
                           whois_info[2],
                           networks_info_txt,
                           arp_info_txt,
                           addons_info_txt)
        except:
            return 'Error! Unable to get system informtaion! :('

# Check if running stand-alone or imported
if __name__ == u'__main__':
    if platform.system() != u'Windows':
        import systeminfo
        si = SystemInfo()
        info = si.get_system_info()
        print(info)
    else:
        print(u'This script does not work with a Windows operating system. :( - Yet!')


import asyncio
import websockets
from concurrent.futures import ThreadPoolExecutor
from enum import Enum
from loguru import logger
import json

from goopy_ibcp.zmq_publisher import ZmqPublisher
from goopy_certificate.certificate import Certificate, CertificateError
from goopy_ibcp.ibmsg_tick import IBMsgConverterTick
from goopy_ibcp.ibmsg import IBMsgConverter
from goopy_ibcp.jsonpacket import JSONPacket
from goopy_ibcp.ibmsg_topic import IBTopic

class ClientPortalWebsocketsError(Enum):
    """Define client portal error messages."""

    Ok = 0
    Unknown = 1
    Invalid_URL = 2
    Invalid_Certificate = 3
    Connection_Failed = 4

# TODO: Document ClientPortalWebsocketsBase class
class ClientPortalWebsocketsBase:
    """
    Interactive Brokers ClientPortal Interface (Websocket).
    Refer to https://interactivebrokers.github.io/cpwebapi/RealtimeSubscription.html for API documentation
    NOTE: Websocket usage also requires the UI to send the /tickle endpoint. See Websocket Ping Session docs.
    """

    def __init__(self):
        # Base used by all IB websocket endpoints
        self.url_ib_wss = "wss://localhost:5000/v1/api/ws"
        self.connection = None
        self.sslcontext = None
        self.msg_handlers = {}
        # default websocket 'tic' heartbeat message is 60 sec
        self.heartbeat_sec = 60
        self.data_subscribers = []
        self.publisher = ZmqPublisher()
        logger.log(
            "DEBUG",
            f"Clientportal (Websockets) Started with endpoint: {self.url_ib_wss}",
        )

        # Message converters we want to handle. Other messages will be discarded when received.
        self.msg_handlers[
            IBTopic.MarketData
        ] = IBMsgConverterTick.create_dict_from_raw_msg

    def loop(self):
        """Start websocket message handler and heartbeat"""
        try:
            with ThreadPoolExecutor() as executor:
                executor.submit(
                    asyncio.get_event_loop().run_until_complete(self.__async_loop())
                )

        except Exception as e:
            logger.log("DEBUG", f"Exception:{e}")

        finally:
            logger.log("DEBUG", f"Exited loop.")

    async def send(self, msg):
        """Websocket send. Automatic logging of transmitted data."""
        if self.connection is not None:
            logger.log("DEBUG", f"{msg}")
            await self.connection.send(msg)

    def on_connection(self, msg):
        """Websocket connection opened. Override as required."""
        pass

    def on_message(self, msg):
        """Websocket message received. Override as required."""
        pass

    async def __open_connection(self, url="", url_validator=None):
        """Open a websocket connection"""
        if url_validator is not None:
            if url_validator(url) is False:
                return ClientPortalWebsocketsError.Invalid_URL

        try:
            logger.log("DEBUG", f"Certificate: Acquiring")
            result = Certificate.get_certificate()

        except Exception as e:
            logger.log("DEBUG", f"EXCEPTION: {e}")
            return ClientPortalWebsocketsError.Unknown

        finally:
            if result.error != CertificateError.Ok:
                logger.log(
                    "DEBUG",
                    f"Certificate: Problems obtaining certificate: {result.error}",
                )
                return ClientPortalWebsocketsError.Invalid_Certificate

        try:
            logger.log("DEBUG", f'Connection: Opening "{url}"')

            self.connection = await websockets.connect(url, ssl=result.ssl_context)
            logger.log("DEBUG", f'Connection: Established "{url}"')
            ret_code = ClientPortalWebsocketsError.Ok

            # Once connection is achieved, IB provides confirmation message with username
            connect_msg = await self.connection.recv()
            logger.log("DEBUG", f"Connection: Confirmation {connect_msg}")

            # additional external handling if overridden
            self.on_connection(connect_msg)

            # save off this context for use by other handlers since it was used successfully already
            self.sslcontext = result.ssl_context

        except (websockets.WebSocketException, TimeoutError) as e:
            logger.log("DEBUG", f"EXCEPTION: Websockets {e}")
            ret_code = ClientPortalWebsocketsError.Connection_Failed

        except Exception as e:
            logger.log("DEBUG", f"EXCEPTION: General exception: {e}")
            ret_code = ClientPortalWebsocketsError.Unknown

        finally:
            return ret_code

    async def __websocket_msg_handler(self):
        logger.log("DEBUG", f"Start message handler")

        async for ws in websockets.connect(self.url_ib_wss, ssl=self.sslcontext):
            try:
                self.connection = ws

                async for msg_raw in ws:
                    logger.log("DEBUG", f"Received {msg_raw}")

                    # Don't know what message type is yet, so just use base and extract the topic
                    new_msg_dict = IBMsgConverter.create_dict_from_raw_msg(msg_raw)
                    new_msg_topic = new_msg_dict[IBTopic.Topic]

                    # If a handler is available, we'll get a specific message dictionary returned to us
                    handled_msg_dict = IBTopic.process_topic(
                        new_msg_topic, self.msg_handlers, msg_raw
                    )

                    # Only transmit packets that were handled
                    if handled_msg_dict is not None:
                        new_json_packet = JSONPacket(new_msg_topic, handled_msg_dict)
                        xmit_msg = new_json_packet.build_packet()
                        self.publisher.publish_string(xmit_msg)
                        logger.log("DEBUG", f"Transmitted: {xmit_msg}")

            except websockets.ConnectionClosed:
                logger.log("DEBUG", f"Connection closed. Re-opening...")
                continue

            except Exception as e:
                logger.log("DEBUG", f"EXCEPTION: {e}")

            finally:
                pass

        logger.log("DEBUG", f"Exited message handler. Restarting.")

    async def __websocket_heartbeat(self):
        if self.connection is not None:
            logger.log("DEBUG", f"Start heartbeat")

            try:
                while True:
                    await self.send("tic")
                    await self.send('smd+551601544+{"fields":["31"]}')
                    logger.log("DEBUG", f"Send Heartbeat {self.heartbeat_sec}s")
                    await asyncio.sleep(self.heartbeat_sec)

            except Exception as e:
                logger.log("DEBUG", f"EXCEPTION: {e}")

            finally:
                logger.log("DEBUG", f"Exited websocket heartbeat")

    async def __websocket_reqdata(self):
        # Subscribe to desired data feeds.
        # Short sleep to let things get connected...TBD needed? Better way?
        await asyncio.sleep(5)
        logger.log("DEBUG", f"Start ReqData")

        # Example request just to get a tick stream started
        await self.send('smd+551601544+{"fields":["31", "84", "86"]}')
        logger.log("DEBUG", f"Sent streaming data request")

        # Example request to initiate historical data gathering
        await self.send(
            'smq+551601544+{"period": "1d","bar": "1min", "source": "trades","format": "%c", "outsideRth":true, "since":"20230510-22:00:00}'
        )
        logger.log("DEBUG", f"Sent historical data request")
        logger.log("DEBUG", f"Exit ReqData")

    async def __async_loop(self):
        # Kick off the various threads
        try:
            logger.log("DEBUG", f"Start Async Loop")
            ret = await self.__open_connection(url=self.url_ib_wss)
            if ret == ClientPortalWebsocketsError.Ok:
                await asyncio.gather(
                    self.__websocket_msg_handler(),
                    self.__websocket_reqdata(),
                    self.__websocket_heartbeat(),
                )

        except Exception as e:
            logger.log("DEBUG", f"EXCEPTION: {e}")

        finally:
            logger.log("DEBUG", f"Exited Async Loop")

if __name__ == "__main__":
    print("=== IB Client Portal Websockets ===")


from django.test import TestCase

from dfirtrack_main.models import Recommendation

class RecommendationModelTestCase(TestCase):
    """recommendation model tests"""

    @classmethod
    def setUpTestData(cls):
        # create object
        Recommendation.objects.create(recommendation_name='recommendation_1')

    def test_recommendation_string(self):
        """test string representation"""

        # get object
        recommendation_1 = Recommendation.objects.get(
            recommendation_name='recommendation_1'
        )
        # compare
        self.assertEqual(str(recommendation_1), 'recommendation_1')

    def test_recommendation_id_attribute_label(self):
        """test attribute label"""

        # get object
        recommendation_1 = Recommendation.objects.get(
            recommendation_name='recommendation_1'
        )
        # get label
        field_label = recommendation_1._meta.get_field('recommendation_id').verbose_name
        # compare
        self.assertEqual(field_label, 'recommendation id')

    def test_recommendation_name_attribute_label(self):
        """test attribute label"""

        # get object
        recommendation_1 = Recommendation.objects.get(
            recommendation_name='recommendation_1'
        )
        # get label
        field_label = recommendation_1._meta.get_field(
            'recommendation_name'
        ).verbose_name
        # compare
        self.assertEqual(field_label, 'recommendation name')

    def test_recommendation_note_attribute_label(self):
        """test attribute label"""

        # get object
        recommendation_1 = Recommendation.objects.get(
            recommendation_name='recommendation_1'
        )
        # get label
        field_label = recommendation_1._meta.get_field(
            'recommendation_note'
        ).verbose_name
        # compare
        self.assertEqual(field_label, 'recommendation note')

    def test_recommendation_name_length(self):
        """test for max length"""

        # get object
        recommendation_1 = Recommendation.objects.get(
            recommendation_name='recommendation_1'
        )
        # get max length
        max_length = recommendation_1._meta.get_field('recommendation_name').max_length
        # compare
        self.assertEqual(max_length, 255)


import datetime
import pymongo

client = pymongo.MongoClient("mongodb://localhost:27017/")
db = client["amazon"]

def add_product_detail(details):

    new = db["products"]
    ASIN = details["url"][len(details["url"])-10:len(details["url"])]

    try:
        details["date"] = datetime.datetime.utcnow()
        new.update_one({"asin":ASIN}, {"$set": {"asin":ASIN}, "$push":{"details":details}}, upsert=True)
        return True
    except Exception as identifier:
        print(identifier)
        return False

def get_product_history(asin):

    new = db["products"]
    try:
        find = new.find_one({"asin": asin}, {"_id": 0})
        if find:
            return find
        else:
            raise Exception("Not found")
    except Exception as identifier:
        print(identifier)
        return None


import geopandas as gpd
import numpy as np
import json
from shapely import geometry
from sklearn.cluster import KMeans
import math

import branca
import folium
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.cm as cm
from collections import defaultdict
from pandas import DataFrame
from matplotlib import animation
from matplotlib.backends.backend_pdf import PdfPages
from mpl_toolkits.axes_grid1 import make_axes_locatable
from designinit import beat_with_max_workload

# merge grids in the same beat to get zone boundary
def get_beat_bound(geo_fname,fname):
    grid_table = np.load("result/grid-%s.npy" % fname)
    grid_gpd = gpd.read_file("data/%s.json" % geo_fname)
    grid_gpd["zone"] = grid_table[:,1]
    beat_boundary_gpd = grid_gpd[['zone','geometry']]
    beat_regions = beat_boundary_gpd.dissolve(by='zone')

    beat_regions.to_file("result/boundary-%s.json" % fname, driver="GeoJSON")
    boundary_geo_fname = "result/boundary-%s.json" % fname
    with open("result/boundary-%s.json" % fname,'r', encoding='utf-8-sig') as f:
        beat_regions_json = json.load(f)

    return beat_regions_json

def visualize_grid(geo_boundary, grid_table, geo_fname, map_fname, min_val=None, max_val=None):
    # center point
    center = [grid_table[:, 4].mean(), grid_table[:, 3].mean()]

    _, beats_set, beats_workload = beat_with_max_workload(grid_table)

    val_dict = defaultdict(lambda: 0)
    for i in range(len(grid_table)):
        grid_id = int(grid_table[i, 0])
        beat_id = grid_table[i, 1]
        val     = beats_workload[beats_set.index(beat_id)] / 3600
#        val_dict[grid_id] = val
        val_dict[grid_id] = math.log(val)

    # map initialization
    _map       = folium.Map(location=center, zoom_start=11, zoom_control=True, max_zoom=17, min_zoom=8)
    # continuous color map intialization
    if min_val is None and max_val is None:
        min_val = min(beats_workload)/3600
        max_val = max(beats_workload)/3600
    cm         = branca.colormap.linear.YlOrRd_09.scale(min_val, max_val) # colorbar for values
    cm.caption = "workload (log)"
    folium.GeoJson(
        data = open("data/%s.json" % geo_fname).read(),
        style_function = lambda feature: {
            'fillColor':   cm(val_dict[int(feature['id'])]),
            'fillOpacity': .5,
            'weight':      0.2,
            'line_opacity':0.5,
            'highlight':   True,}).add_to(_map)

    folium.GeoJson(geo_boundary, name='beat boundary',style_function=lambda feature: {
        'color': 'grey',
        'fillOpacity': 0,
        'weight': 1.7
        }).add_to(_map)

    _map.add_child(cm)
    # save the map
    _map.save("result/map-%s.html" % map_fname)

if __name__ == "__main__":
    # load data
#    greedy_fname = "regression-workload-2020-nbeat-15"
#    redesign_fname   = "redesign-regression-workload-2020-nbeat-15"

    greedy_fname   = "regression-workload-2021-nbeat-15"
    redesign_fname = "redesign-regression-workload-2021-nbeat-15"

    greedy_design  = np.load("result/grid-%s.npy" % greedy_fname)
    redesign  = np.load("result/grid-%s.npy" % redesign_fname)

    grid_fname = "grids"

    greedy_geo = get_beat_bound(grid_fname,greedy_fname)
    redesign_geo = get_beat_bound(grid_fname,redesign_fname)

    _, _, init_beats_workload  = beat_with_max_workload(greedy_design)
    _, _, final_beats_workload  = beat_with_max_workload(redesign)

    ex_fname = "Jan-APR-2019-PD"
    ex_design = np.load("data/grid-%s.npy" % ex_fname)
    # ex_geo = get_beat_bound(grid_fname,ex_fname)
    _, _, ex_beats_workload  = beat_with_max_workload(ex_design)

#    cm_min_val = min([min(init_beats_workload)/3600, min(final_beats_workload)/3600])
#    cm_max_val = max([max(init_beats_workload)/3600, max(final_beats_workload)/3600])

    cm_min_val = math.log(min([min(init_beats_workload)/3600, min(final_beats_workload)/3600, min(ex_beats_workload)/3600]))
    cm_max_val = math.log(max([max(init_beats_workload)/3600, max(final_beats_workload)/3600, max(ex_beats_workload)/3600]))
    print(cm_min_val,cm_max_val)

    visualize_grid(greedy_geo, greedy_design, grid_fname, map_fname=greedy_fname,min_val=cm_min_val,max_val=cm_max_val)
    visualize_grid(redesign_geo, redesign, grid_fname, map_fname=redesign_fname,min_val=cm_min_val,max_val=cm_max_val)

    # visualize_grid(ex_geo, ex_design, grid_fname, map_fname=ex_fname,min_val=cm_min_val,max_val=cm_max_val)
#    visualize_grid(ex_geo, ex_design, grid_fname, map_fname=ex_fname)


# -*- coding: utf-8 -*-

__author__ = 'Robert Zeng'
__updated__ = '18.08.2018'  # day.month.year

import logging

"""
All objects imported here are exposed as the public API of GoogleScraper
"""

logging.getLogger(__name__)


#Importing packages
import streamlit as st
import numpy as np
import os
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from keras.applications.inception_v3 import InceptionV3
import cv2 as cv
from PIL import Image

classes = ['Pepper__bell___healthy',
 'Potato___Early_blight',
 'Potato___healthy',
 'Potato___Late_blight',
 'Tomato__Target_Spot',
 'Tomato__Tomato_mosaic_virus',
 'Tomato__Tomato_YellowLeaf__Curl_Virus',
 'Tomato_Bacterial_spot',
 'Tomato_Early_blight',
 'Tomato_healthy',
 'Tomato_Late_blight',
 'Tomato_Leaf_Mold',
 'Tomato_Septoria_leaf_spot',
 'Tomato_Spider_mites_Two_spotted_spider_mite',
 'Pepper__bell___Bacterial_spot']

#Title page
st.title("WebApp")

#Image Uploader
uploaded_file = st.file_uploader("Choose a file")
if uploaded_file is not None:

    # pillow opening image and displaying image
    image = Image.open(uploaded_file)
    st.image(image)

    # resizing image with cv2
    img_2rgb = image.convert("RGB")
    np_array = np.array(img_2rgb)
    final_image = cv.resize(np_array,(75,75))

    # expanding image to fit to input into the model
    expanded_img = np.expand_dims(final_image,axis = 0)

    #loading in the model
    model = tf.keras.models.load_model('tl_secondpass-20220125T010409Z-001/tl_secondpass')

    # prediction with the model
    prediction = (model.predict(expanded_img))
    prediction_index = np.argmax(prediction)
    final_prediction = classes[prediction_index]
    confidence = str(prediction[0][prediction_index]*100)

    st.write("The model predicts this image is a", final_prediction,"with a confidence of",confidence+'%')


import torch
import torch.nn as nn
import torch.nn.functional as F

#计算attention
class ScaledDotProductAttention(nn.Module):
    ''' Scaled Dot-Product Attention '''

    def __init__(self, temperature, attn_dropout=0.1):
        super().__init__()
        # 其实就是论文中的根号d_k
        self.temperature = temperature
        # temperature = d_k ** 0.5
        self.dropout = nn.Dropout(attn_dropout)

    def forward(self, q, k, v, mask=None):
        # sz_b: batch_size 批量大小
        # len_q,len_k,len_v: 序列长度 在这里他们都相等
        # n_head: 多头注意力 默认为8
        # d_k,d_v: k v 的dim(维度) 默认都是64
        # 此时q的shape为(sz_b, n_head, len_q, d_k) (sz_b, 8, len_q, 64)
        # 此时k的shape为(sz_b, n_head, len_k, d_k) (sz_b, 8, len_k, 64)
        # 此时v的shape为(sz_b, n_head, len_k, d_v) (sz_b, 8, len_k, 64)
        # q先除以self.temperature(论文中的根号d_k) k交换最后两个维度(这样才可以进行矩阵相乘) 最后两个张量进行矩阵相乘

        # attn的shape为(sz_b, n_head, len_q, len_k)
        attn = torch.matmul(q / self.temperature, k.transpose(2, 3))

        #decoder中的mask
        if mask is not None:
            # 用-1e9代替0 -1e9是一个很大的负数 经过softmax之后接近与0
            # 其一：去除掉各种padding在训练过程中的影响
            # 其二，将输入进行遮盖，避免decoder看到后面要预测的东西。（只用在decoder中）
            attn = attn.masked_fill(mask == 0, -1e9)
            #mask==0取值为true使用-1e9填充

        # 先在attn的最后一个维度做softmax 再dropout 得到注意力分数
        attn = self.dropout(F.softmax(attn, dim=-1))
        # 最后attn与v进行矩阵相乘
        # output的shape为(sz_b, 8, len_q, 64)
        output = torch.matmul(attn, v)
        # 返回 output和注意力分数
        return output, attn


import matplotlib.pyplot as plt
import numpy as np

fig, ax = plt.subplots(2, 1, figsize=(10, 6), tight_layout=True,
                       facecolor=(.18, .31, .31))
x = np.arange(10)
classes = ['plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']
cmap = plt.get_cmap('tab10')
for idx, c in enumerate(classes):
    ax[0].plot(x, idx * x)
    ax[1].plot(x, idx * x, color=cmap(idx), linestyle='dotted', label=c)
ax[1].set_title('color map', color='0.7')
ax[1].legend(loc='best', fancybox=True, ncol=5)
for axes in ax:
    axes.tick_params(labelcolor='tab:orange')
plt.tight_layout(pad=0.5)
plt.show()


from flask import Flask, render_template
import csv, os

def load_scores(filename):
    with open(filename, newline='') as csvfile:
        reader = csv.DictReader(csvfile)
        players = []
        for line in reader:
            players.append(line)
    return players

def process_scores(players):
    dict_list = []
    for player in players:
        score_dict = {}
        score_dict["name"] = player["name"]
        score_dict["id"] = player["id"]
        score_dict["level"] = player["level"]
        score_dict["score"] = player["score"]
        dict_list.append(score_dict)
    return dict_list

def get_scores():
    filename = 'pvzscore.csv'
    data = load_scores(filename)
    scores = process_scores(data)
    return scores

app = Flask(__name__)

@app.route("/")
def home():
    return render_template("home.html")

@app.route("/about")
def about():
    return render_template("about.html")

@app.route("/scoreboard")
def score():
    scores = get_scores()
    return render_template("scoreboard.html", scores = scores)

@app.route("/scoreboard/<int:player_id>")
def player(player_id):
    scores = get_scores()
    for score in scores:
        if score["id"] == str(player_id):
            return render_template("player.html", score = score)

@app.route("/units")
def units():
    return render_template("units.html")

if __name__ == "__main__":
    port = int(os.environ.get("PORT", 5000))
    app.debug = True
    app.run(host='0.0.0.0', port=port)


"""Autocompletion functionality of epigraphdb entities
"""
from typing import Any, Callable, List

import requests
from elasticsearch import Elasticsearch
from elasticsearch.helpers import bulk

from app.settings import api_key, api_url
from app.utils import api_request_headers, batch_by_n
from app.utils.logging import logger

from .indexers import name_indexer_edge_ngram_long

def index_data(
    input_data: List[Any],
    index_name: str,
    es_client: Elasticsearch,
    indexer_fn: Callable,
    batch_size: int = 500,
    overwrite: bool = True,
) -> None:
    logger.info(f"Creating the index {index_name}.")
    indexer_fn(index_name=index_name, overwrite=overwrite, es_client=es_client)

    # NOTE: batch_data is a generator
    batch_data = batch_by_n(collection=input_data, batch_size=batch_size)
    for batch_docs in batch_data:
        requests = [
            dict(**{"_op_type": "index", "_index": index_name}, **doc)
            for doc in batch_docs
        ]
        bulk(es_client, requests)

    es_client.indices.refresh(index=index_name, request_timeout=300)
    logger.info("Done indexing.")

def index_ac_name_from_cypher(
    cypher_query: str,
    index_name: str,
    es_client: Elasticsearch,
    indexer_fn: Callable = name_indexer_edge_ngram_long,
    overwrite: bool = False,
) -> bool:
    """Index entity names from upstream cypher query.

    Cypher results need to return a data structure like:
    [
       { "name": ... },
       { "name": ... },
    ]
    """
    if not es_client.indices.exists(index=index_name) or overwrite:
        cypher_url = f"{api_url}/raw_cypher/"
        r = requests.get(
            cypher_url,
            params={"query": cypher_query, "api_key": api_key},
            headers=api_request_headers,
        )
        r.raise_for_status()
        input_data = [dict(name=item["name"]) for item in r.json()["results"]]
        index_data(
            input_data=input_data,
            index_name=index_name,
            es_client=es_client,
            indexer_fn=indexer_fn,
        )
    return True


import os
import librosa
import argparse
import numpy as np
from tqdm import tqdm
from concurrent.futures import ThreadPoolExecutor, as_completed
from scipy.io import wavfile

def resample_wave(wav_in, wav_out, sample_rate):
    wav, _ = librosa.load(wav_in, sr=sample_rate)
    wav = wav / np.abs(wav).max() * 0.6
    wav = wav / max(0.01, np.max(np.abs(wav))) * 32767 * 0.6
    wavfile.write(wav_out, sample_rate, wav.astype(np.int16))

def process_file(file, wavPath, spks, outPath, sr):
    if file.endswith(".wav"):
        file = file[:-4]
        resample_wave(f"{wavPath}/{spks}/{file}.wav", f"{outPath}/{spks}/{file}.wav", sr)

def process_files_with_thread_pool(wavPath, spks, outPath, sr, thread_num=None):
    files = [f for f in os.listdir(f"./{wavPath}/{spks}") if f.endswith(".wav")]

    with ThreadPoolExecutor(max_workers=thread_num) as executor:
        futures = {executor.submit(process_file, file, wavPath, spks, outPath, sr): file for file in files}

        for future in tqdm(as_completed(futures), total=len(futures), desc=f'Processing {sr} {spks}'):
            future.result()

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("-w", "--wav", help="wav", dest="wav", required=True)
    parser.add_argument("-o", "--out", help="out", dest="out", required=True)
    parser.add_argument("-s", "--sr", help="sample rate", dest="sr", type=int, required=True)
    parser.add_argument("-t", "--thread_count", help="thread count to process, set 0 to use all cpu cores", dest="thread_count", type=int, default=1)

    args = parser.parse_args()
    print(args.wav)
    print(args.out)
    print(args.sr)

    os.makedirs(args.out, exist_ok=True)
    wavPath = args.wav
    outPath = args.out

    assert args.sr == 16000 or args.sr == 32000

    for spks in os.listdir(wavPath):
        if os.path.isdir(f"./{wavPath}/{spks}"):
            os.makedirs(f"./{outPath}/{spks}", exist_ok=True)
            if args.thread_count == 0:
                process_num = os.cpu_count() // 2 + 1
            else:
                process_num = args.thread_count
            process_files_with_thread_pool(wavPath, spks, outPath, args.sr, process_num)