import sys
import copy
import time

import pandas as pd
import numpy as np
import matplotlib.pylab as plt

from helpers.file_system import write_to_json, write_to_feather
from parameter.mcmc.helpers import compute_acf
from parameter.mcmc.helpers import compute_iact
from parameter.mcmc.helpers import compute_sjd

from palettable.colorbrewer.qualitative import Dark2_8

def print_progress_report(mcmc, proposed_state, state_history, lag=1000):
    """Prints the progress report to screen."""

    i = mcmc.current_iter
    iters_between_reports = mcmc.settings['no_iters_between_progress_reports']

    if (i + 1) % iters_between_reports is not 0:
        return None

    mean_time_per_iter = (time.time() - mcmc.start_time) / i
    est_time_remaining = mean_time_per_iter * (mcmc.settings['no_iters'] - i)

    avg_params = np.zeros(mcmc.no_params_to_estimate)
    for j in range(i):
        avg_params += state_history[j]['params'] / (i + 1)

    avg_accept_rate = 0.0
    if i > (lag + 1):
        for j in range(i - lag + 1, i):
            avg_accept_rate += state_history[j]['accepted'] / float(lag)
    else:
        for j in range(i):
            avg_accept_rate += state_history[j]['accepted'] / (i + 1)

    print("###################################################################")
    print(" Iteration: " + str(i + 1) + " of : "
        + str(mcmc.settings['no_iters']) + " completed.")
    print(" Time per iteration: {:.3f}s and estimated time remaining: {:.3f}s.".format(mean_time_per_iter, est_time_remaining))
    print("")
    print(" Current state of the Markov chain:")
    print( ["%.4f" % v for v in state_history[i-1]['params']])
    print("")
    print(" Proposed next state of the Markov chain:")
    print( ["%.4f" % v for v in proposed_state['params']])
    print("")
    print(" Current posterior mean estimate: ")
    print( ["%.4f" % v for v in avg_params])
    print("")
    print(" Current acceptance rate (over last 1,000 iterations):")
    print(" %.4f" % avg_accept_rate)
    print("")
    print(" Current IACT values (over last 1,000 iterations):")
    print( ["%.2f" % v for v in compute_iact(mcmc, state_history, lag=lag)])
    if mcmc.alg_type is 'qmh':
        if (i > mcmc.settings['memory_length']):
            avg_samples_hessian = 0.0
            avg_hessian_estimated = 0.0
            avg_hessians_corrected = 0.0
            for j in range(i):
                if state_history[j]['hessian_samples'] > 0:
                    avg_samples_hessian += state_history[j]['hessian_samples'] / i
                    avg_hessian_estimated += state_history[j]['hessian_estimated'] * 1.0 / i
                    avg_hessians_corrected += state_history[j]['hessian_corrected'] * 1.0 / i
            print("")
            print(" Mean number of samples for Hessian estimate:")
            print(" {:.4f}".format(avg_samples_hessian))
            print("")
            print(" Rate of successful QN proposal computations:")
            print(" {:.4f}".format(avg_hessian_estimated))
            print("")
            print(" Rate of Hessian correction:")
            print(" {:.4f}".format(avg_hessians_corrected))
    if 'adapted_step_size' in proposed_state:
        print("")
        print(" Current adapted step size (with target accept prob: {}):".format(mcmc.settings['adapt_step_size_target']))
        print(" {:.4f}".format(proposed_state['adapted_step_size']))
    print("")
    print("###################################################################")
    print("")

def print_greeting(mcmc, estimator):
    """Prints the greeting when starting a run to screen."""

    print("")
    print("###################################################################")
    print("Starting MH algorithm...")
    print("")
    print("Sampling from the parameter posterior using: " + mcmc.name)
    print("in the model: " + mcmc.model.name)
    print("")
    if mcmc.using_gradients:
        print("Likelihood and gradients estimated using:")
        print(estimator.name)
        if mcmc.hessians_computed_by:
            print("Hessian is estimated using: " + mcmc.hessians_computed_by)
    else:
        print("Likelihood estimated using:")
        print(estimator.name)
    print("")
    if 'hessian' in mcmc.settings:
        step_size = mcmc.settings['step_size_hessian']
        eff_step_size = np.sqrt(np.diag(mcmc.settings['hessian'])) * step_size
        print("The proposal has effective random step sizes: ")
        i = 0
        for param in mcmc.model.params_to_estimate:
            print("{}: {:.3f}".format(param, eff_step_size[i]))
            i += 1
        print("")
    print("Running MH for {} iterations.".format(mcmc.settings['no_iters']))
    print("")
    print("The rest of the settings are as follows: ")
    for key in mcmc.settings:
        print("{}: {}".format(key, mcmc.settings[key]))
    print("")
    print("###################################################################")

def plot(mcmc, max_acf_lag = 500):
    """ Plots results to the screen after a run of an MCMC algorithm. """
    no_iters = mcmc.settings['no_iters']
    no_burnin_iters = mcmc.settings['no_burnin_iters']
    effective_iters = no_iters - no_burnin_iters
    no_params = mcmc.model.no_params_to_estimate
    param_names = mcmc.model.params_to_estimate
    no_bins = int(np.sqrt(effective_iters))

    accept_rate = params = np.zeros(effective_iters)
    params = np.zeros((effective_iters, no_params))
    prop_params = np.zeros((effective_iters, no_params))
    nat_gradient = np.zeros((effective_iters, no_params))
    no_nat_gradients = True

    for i in range(effective_iters):
        j = i + no_burnin_iters
        accept_rate[i] = mcmc.state_history[j]['accepted']
        params[i, :] = mcmc.state_history[j]['params']
        prop_params[i, :] = mcmc.state_history[j]['params_prop']
        if 'nat_gradient' in mcmc.state_history[j]:
            nat_gradient[i, :] = mcmc.state_history[j]['nat_gradient']
            no_nat_gradients = False
        else:
            no_nat_gradients = True

    accept_rate = np.cumsum(accept_rate) / np.arange(1, effective_iters+1)

    plt.figure(1)
    plt.plot(accept_rate)
    plt.ylabel("Running mean acceptance rate")
    plt.xlabel("iter")

    plt.figure(2)
    for i in range(no_params):
        col = Dark2_8.mpl_colors[i % 8]

        plt.subplot(no_params, 5, 5 * i + 1)
        plt.hist(params[:, i], bins=no_bins, color = col)
        plt.ylabel("Marginal posterior probability of " + str(param_names[i]))
        plt.xlabel("iter")

        plt.subplot(no_params, 5, 5 * i + 2)
        plt.plot(params[:, i], color = col)
        plt.ylabel("Parameter trace of " + str(param_names[i]))
        plt.xlabel("iter")

        plt.subplot(no_params, 5, 5 * i + 3)
        acf = compute_acf(params[:, i], max_lag=max_acf_lag)
        plt.plot(np.arange(max_acf_lag), acf, color = col)
        plt.ylabel("ACF of " + str(param_names[i]))
        plt.xlabel("iter")

        plt.subplot(no_params, 5, 5 * i + 4)
        plt.plot(prop_params[:, i], color = col)
        plt.ylabel("Proposed trace of " + str(param_names[i]))
        plt.xlabel("iter")

        plt.subplot(no_params, 5, 5 * i + 5)
        if not no_nat_gradients:
            plt.plot(nat_gradient[:, i], color = col)
            plt.ylabel("natural gradient of " + str(param_names[i]))
            plt.xlabel("iter")
    plt.show()

def compile_benchmark_results(mcmc, sim_name=None):
    """Helper for compiling the benchmark results from a run of
       the algorithm in mh_quasi_newton_benchmark.py before saving to file."""

    no_iters = mcmc.settings['no_iters']
    no_burnin_iters = mcmc.settings['no_burnin_iters']
    no_effective_iters = no_iters - no_burnin_iters
    idx = range(no_burnin_iters, no_iters)
    current_time = time.strftime("%c")

    if not 'error_sr1_fro' in mcmc.state_history[no_iters-1]:
        return None

    benchmarkout = {}
    benchmarkout.update({'simulation_name': sim_name})
    benchmarkout.update({'simulation_time': current_time})
    benchmarkout.update({'time_per_iteration': mcmc.time_per_iter})

    df = pd.DataFrame(mcmc.state_history)
    field_names = ['error_bfgs_fro', 'error_ls_fro', 'error_sr1_fro']

    for field in field_names:
        if not field in mcmc.state_history[no_iters-1]:
            print("Skipping saving " + field + " to file.")
            continue

        if type(df[no_iters-1][field]) is float or type(df[no_iters-1][field]) is np.float64:
            tmp = np.zeros((no_effective_iters, 1))
        else:
            tmp = np.zeros((no_effective_iters, len(df[no_iters-1][field])))

        for i in range(no_effective_iters):
            foo = df[i + no_burnin_iters][field]

            if type(foo) is not np.ndarray:
                if np.isnan(foo): foo = 0.0
                if np.isposinf(foo): foo = 10000000.0
                if np.isneginf(foo): foo = -10000000.0

            if type(foo) is np.ndarray:
                idx = np.where(np.isnan(foo))
                foo[idx] = 0.0
                idx = np.where(np.isposinf(foo))
                foo[idx] = 10000000.0
                idx = np.where(np.isneginf(foo))
                foo[idx] = -10000000.0

            tmp[i, :] = foo

        benchmarkout.update({field: tmp})

    return benchmarkout

def compile_results(mcmc, sim_name=None):
    """Helper for compiling the results from a run before saving to file."""
    no_iters = mcmc.settings['no_iters']
    no_burnin_iters = mcmc.settings['no_burnin_iters']
    no_effective_iters = no_iters - no_burnin_iters
    idx = range(no_burnin_iters, no_iters)
    current_time = time.strftime("%c")

    mcmcout = {}
    mcmcout.update({'simulation_name': sim_name})
    mcmcout.update({'simulation_time': current_time})
    mcmcout.update({'time_per_iteration': mcmc.time_per_iter})

    df = pd.DataFrame(mcmc.state_history)
    field_names = ['params', 'params_prop', 'nat_gradient', 'accepted', 'state_trajectory']

    for field in field_names:
        if not field in mcmc.state_history[0]:
            print("Skipping saving " + field + " to file.")
            continue

        if type(df[0][field]) is float or type(df[0][field]) is np.float64:
            tmp = np.zeros((no_effective_iters, 1))
        else:
            tmp = np.zeros((no_effective_iters, len(df[0][field])))

        for i in range(no_effective_iters):
            foo = df[i + no_burnin_iters][field]

            if type(foo) is not np.ndarray:
                if np.isnan(foo): foo = 0.0
                if np.isposinf(foo): foo = 10000000.0
                if np.isneginf(foo): foo = -10000000.0

            if type(foo) is np.ndarray:
                idx = np.where(np.isnan(foo))
                foo[idx] = 0.0
                idx = np.where(np.isposinf(foo))
                foo[idx] = 10000000.0
                idx = np.where(np.isneginf(foo))
                foo[idx] = -10000000.0

            tmp[i, :] = foo

        mcmcout.update({field: tmp})

    if hasattr(mcmc, 'adapted_step_sizes'):
        mcmcout.update({'adapted_step_sizes': mcmc.adapted_step_sizes})

    if hasattr(mcmc, 'no_hessians_corrected'):
        mcmcout.update({'no_hessians_corrected': mcmc.no_hessians_corrected})

    data = {}
    data.update({'observations': mcmc.model.obs})
    if mcmc.model.states is None:
        data.update({'states': mcmc.model.states})
    data.update({'simulation_name': sim_name})
    data.update({'simulation_time': current_time})

    settings = copy.deepcopy(mcmc.settings)
    recasted_settings = {}
    for key in settings:
        recasted_settings.update({key: str(settings[key])})
    recasted_settings.update({'sampler_name': mcmc.name})
    recasted_settings.update({'simulation_name': sim_name})
    recasted_settings.update({'simulation_time': current_time})

    return mcmcout, data, recasted_settings, df

def save_to_file(mcmc, file_path, sim_name=None, sim_desc=None):
    """Helper for saving results to file."""
    mcout, data, settings, mcoutdf = compile_results(mcmc, sim_name=sim_name)

    if sim_desc:
        desc = {'description': sim_desc,
                'time': settings['simulation_time']
            }
        write_to_json(desc, file_path, sim_name, 'description.txt')

    try:
        write_to_json(mcout, file_path, sim_name, 'mcmc_output.json')
    except:
        print("Error exporting JSON. Probably due to NaN or Inf. Use feather instead.")
    #write_to_feather(mcoutdf, file_path, sim_name, 'mcmc_output.feather')
    write_to_json(data, file_path, sim_name, 'data.json')
    write_to_json(settings, file_path, sim_name, 'settings.json')

    benchmark = compile_benchmark_results(mcmc, sim_name=sim_name)

    if not type(benchmark) is type(None):
        write_to_json(benchmark, file_path, sim_name, 'benchmark.json')


import json

# 設定 JSON 檔案的相對路徑
file_path = "F-C0032-001.json"

# 讀取 JSON 檔案內容
with open(file_path, "r") as file:
    data_json = json.load(file)

location = data_json["cwbopendata"]["dataset"]["location"]  # 取出 location 的內容
for i in location:
    print(f"{i}")


import os
import azure.functions as func
import boto3
from botocore.exceptions import ClientError

def main(req: func.HttpRequest) -> func.HttpResponse:
    username = req.params.get('username')
    policyarn = req.params.get('policyarn')
    if username and policyarn:
        try:
            iam = boto3.client('iam',
                               aws_access_key_id=os.environ["AWS_AccessKeyId"],
                               aws_secret_access_key=os.environ["AWS_SecretAccessKey"])
            try:
                iam.detach_user_policy(UserName=username, PolicyArn=policyarn)
            except iam.exceptions.NoSuchEntityException or iam.exceptions.LimitExceededException or \
                   iam.exceptions.InvalidInputException as err:
                return func.HttpResponse(str(err), status_code=404)
            return func.HttpResponse("Successfully detached the user policy.", status_code=200)
        except ClientError as err:
            return func.HttpResponse(str(err), status_code=401)
    else:
        return func.HttpResponse("Please pass a username and a policyarn on the query string", status_code=400)


import asyncio
import datetime
import hashlib
import logging
import os
from dataclasses import dataclass
from functools import partial
from typing import Dict, Callable, Optional, Awaitable

import bs4
import httpx
import sanic
from anyio import create_task_group
from sanic.request import Request
from sanic.response import text

logger = logging.getLogger("websub")

class HubNotFoundException(Exception):
    pass

@dataclass
class Subscription:
    hub: str
    topic: str
    hex_id: str

    handler: Callable[[bytes], Awaitable[None]]

    verified_at: Optional[datetime.datetime] = None
    lease: Optional[datetime.timedelta] = None

    cold: bool = True

    def __repr__(self) -> str:
        return f"<Subscription {self.topic} (0x{self.hex_id.upper()} {self.lease})>"

class WebSubClient:
    def __init__(self, https: bool, app: sanic.Sanic, server: str = None, _from: str = None):
        self.server = server
        self.https = https
        self.app = app
        self._from = f"{_from or 'default'} (pywebsub)"
        self.running = False
        self.subscriptions: Dict[str, Subscription] = {}
        self.seen_hashes = set()
        self.heartbeat_hashes = set()

    async def subscribe(self, hub: str, topic: str, handler: Callable[[bytes], Awaitable[None]]):
        s = Subscription(hub=hub,
                         topic=topic,
                         hex_id=os.urandom(16).hex(),
                         handler=handler)
        self.subscriptions[topic] = s
        if self.running:
            s.cold = False
            await self.make_request(s)

    async def unsubscribe(self, topic: str):
        sub = self.subscriptions.get(topic)
        if sub is None:
            logger.warning(f"Unsubscribe requested for {topic}, but subscription does not exist.")
        else:
            if self.running:
                await self.make_request(sub, "unsubscribe")
            del self.subscriptions[topic]

    async def discover(self, topic: str) -> str:
        try:
            resp = await httpx.get(topic, headers=self.headers)
        except httpx.exceptions.HTTPError as e:
            logger.exception(f"{topic} discovery request failed with exception:")
            raise HubNotFoundException(e)

        if resp.status_code not in range(200, 300):
            raise HubNotFoundException(f"{topic} discovery request failed, status is {resp.status_code}")
        else:
            logger.debug(f"{topic} discovery request succeeded")
            soup = bs4.BeautifulSoup(resp.content, "xml")
            link = soup.find("link", rel="hub")
            if link is None:
                raise HubNotFoundException(f"rel=hub link not present in {topic}")

            return link.attrs['href']

    async def discover_and_sub(self, topic: str, handler: Callable[[bytes], Awaitable[None]]):
        hub = await self.discover(topic)
        await self.subscribe(hub, topic, handler)

    async def make_request(self, sub: Subscription, mode: str = "subscribe"):
        callback_url = self.make_callback_url(sub.hex_id)
        logger.info(f"Subscribing to {sub.topic}, calling back to {callback_url}")

        values = {
              "hub.callback": callback_url,
              "hub.topic":    sub.topic,
              "hub.mode":     "subscribe"
        }

        try:
            resp = await httpx.post(sub.hub, data=values, headers=self.headers)
        except httpx.exceptions.HTTPError:
            logger.exception(f"{mode} request failed with exception, {sub} at {callback_url}:")
            return

        if resp.status_code != 202:
            logger.error(f"{mode} request failed, {sub} at {callback_url}, status is {resp.status_code}")
        else:
            logger.info(f"{mode} request succeeded, {sub} at {callback_url}")

    def make_callback_url(self, hex_id: str) -> str:
        return self.url_for('callback', hex_id=hex_id)

    def url_for(self, view_name: str, **kwargs):
        return self.app.url_for(f"websub.{view_name}", **kwargs,
                                _scheme='https' if self.https else 'http',
                                _external=True, _server=self.server)

    def get_sub_by_hex(self, hex_id: str) -> Optional[Subscription]:
        for s in self.subscriptions.values():
            if s.hex_id.lower() == hex_id.lower():
                return s

    async def broadcast(self, sub: Subscription, req: Request):
        body: bytes = req.body
        h: bytes = hashlib.md5(req.body).digest()
        if h not in self.seen_hashes:
            self.seen_hashes.add(h)
            await sub.handler(body)
        else:
            logger.info(f"Update for {sub} already handled, hash is {h.hex()}")

    @property
    def bp(self) -> sanic.Blueprint:
        bp = sanic.Blueprint("websub")

        @bp.route("/hb/<hex_id>")
        async def heartbeat(request: Request, hex_id: str) -> sanic.response.HTTPResponse:
            return text(hex_id, status=200 if hex_id in self.heartbeat_hashes else 404)

        @bp.route("/push-callback/<hex_id>", methods=["GET", "POST"])
        async def callback(request: Request, hex_id: str) -> sanic.response.HTTPResponse:
            topic = request.args.get("hub.topic")
            mode = request.args.get("hub.mode")

            if mode == "subscribe":
                sub = self.subscriptions.get(topic)
                if sub is None:
                    logger.warning(f"Unexpected subscription for {topic} on {hex_id}")
                    return text("Unexpected subscription", status=400)
                else:
                    lease_s = request.args.get('hub.lease_seconds')
                    if not lease_s:
                        logger.error(f"Lease is not resolved for sub {sub}: {lease_s!r}")
                        return text("No lease", status=400)
                    sub.verified_at = datetime.datetime.now()
                    sub.lease = datetime.timedelta(seconds=int(lease_s))
                    logger.info(f"Subscription verified for {sub}, lease is {sub.lease}")
                    if hex_id != sub.hex_id:
                        logger.warning(f"Subscription hex does not match path hex: {sub.hex_id} != {hex_id}")
                    return text(request.args.get("hub.challenge"))

            elif mode == "unsubscribe":
                sub = self.subscriptions.get(topic)
                if sub is None:
                    logger.warning(f"Unexpected unsubscribe for {topic} on {hex_id}")
                    return text("Unexpected unsubscribe", status=400)
                else:
                    logger.info(f"Unsubscribe confirmed for {topic} on {hex_id}")
                    if hex_id != sub.hex_id:
                        logger.warning(f"Subscription hex does not match path hex: {sub.hex_id} != {hex_id}")
                    return text(request.args.get("hub.challenge"))

            elif mode == "denied":
                logger.warning(f"Subscription denied for {topic}, reason was {request.args.get('hub.reason')}")
                return sanic.response.HTTPResponse()
                # TODO: Don't do anything for now, should probably mark the subscription.
            else:
                sub = self.get_sub_by_hex(hex_id)
                if sub is None:
                    logger.warning("Got unknown message for unknown subscription:")
                    logger.warning(f"{hex_id}: {request}")
                    return text("Unknown subscription", status=400)
                else:
                    logger.info(f"Update for {sub}")
                    await self.broadcast(sub, request)
                    return sanic.response.HTTPResponse()

        return bp

    @property
    def headers(self):
        return {
              "From": self._from
        }

    def install(self):
        bp = self.bp
        if bp.name not in self.app.blueprints:
            self.app.blueprint(self.bp)

    async def boot(self, *args, **kwargs):
        create_server = partial(self.app.create_server, *args, **kwargs, return_asyncio_server=True)

        async with create_task_group() as tg:
            await tg.spawn(create_server)
            await tg.spawn(self.start)

    async def start(self):
        self.running = True
        self.install()
        logger.debug("waiting 5 seconds before starting checking wait_till_heartbeat...")
        await asyncio.sleep(5)
        logger.debug("waiting till heartbeat...")
        await self.wait_till_heartbeat()
        logger.debug("starting subbed loop...")
        await self.ensure_subbed_loop()

    async def wait_till_heartbeat(self):
        class NotCorrect(Exception):
            pass

        while True:
            try_hex = os.urandom(6).hex()
            self.heartbeat_hashes.add(try_hex)
            try:
                r = await httpx.get(self.url_for("heartbeat", hex_id=try_hex))
                if r.status_code != 200:
                    raise NotCorrect()
            except httpx.exceptions.HTTPError:
                logger.debug("heartbeat failed with HTTPError, waiting 2 seconds...")
                await asyncio.sleep(2)
            except NotCorrect:
                logger.debug("heartbeat failed reaching current server, waiting 2 seconds...")
                await asyncio.sleep(2)
            else:
                return
            finally:
                self.heartbeat_hashes.remove(try_hex)

    async def ensure_subbed_loop(self):
        while True:
            try:
                await self.ensure_subbed()
            except Exception:
                logger.exception("ensure_subbed gave an exception")

            await asyncio.sleep(60)

    async def ensure_subbed(self):
        logger.debug("ensure_subbed called")
        in_one_hour = datetime.datetime.now() + datetime.timedelta(hours=1)
        for s in self.subscriptions.values():
            logger.debug(f"ensure_subbed: checking {s}")
            if not s.cold and (s.verified_at is None or s.lease is None):
                logger.debug(f"ensure_subbed: {s} not yet verified, skipping...")
                continue
            if s.cold or (s.verified_at + s.lease) < in_one_hour:
                s.cold = False
                await self.make_request(s)
            else:
                logger.debug(f"ensure_subbed: {s} does not need new lease, verified at {s.verified_at}")


"""add teams table

Revision ID: 142624f4f7a0
Revises: bcb16e8d296b
Create Date: 2022-12-05 22:26:34.032944

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '142624f4f7a0'
down_revision = 'bcb16e8d296b'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('teams',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('name', sa.String(length=50), nullable=False),
    sa.PrimaryKeyConstraint('id')
    )
    with op.batch_alter_table('candidates', schema=None) as batch_op:
        batch_op.add_column(sa.Column('team_id', sa.Integer(), nullable=True))
        batch_op.create_foreign_key(None, 'teams', ['team_id'], ['id'])

    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    with op.batch_alter_table('candidates', schema=None) as batch_op:
        batch_op.drop_constraint(None, type_='foreignkey')
        batch_op.drop_column('team_id')

    op.drop_table('teams')
    # ### end Alembic commands ###


import sys
sys.path.insert(0, '..')
import argparse
import cv2
import numpy as np
from os import path
import os
from numpy.random import default_rng
from tqdm import tqdm
from utils.misc import find_iou_bulk, create_bbr_annotation_v2_bulk, crop_and_resize_v2
from PNet.model import pnet_predict, pnet_predict_v2
from RNet.model import rnet_predict
import tensorflow as tf
from matplotlib import pyplot as plt
import matplotlib
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'

def get_arguments():
    """
    Get arguments from the command line.
    """
    parser = argparse.ArgumentParser(description='Create Positives, Negatives and Part Faces from the WIDER FACE dataset for R-Net.')
    parser.add_argument(
        '-a',
        '--annotation-file',
        type=str,
        help='The path to the annotation file of the WIDER FACE dataset',
        required=True
    )
    parser.add_argument(
        '-i',
        '--images-directory',
        type=str,
        help='The path to the folder that contains WIDER FACE dataset',
        required=True
    )
    parser.add_argument(
        '-o',
        '--output-directory',
        type=str,
        help='The path to the folder where you want to save your output',
        default='./data/temp'
    )
    parser.add_argument(
        '--pnet',
        type=str,
        help='The path to the .hdf5 file containing the weights of P-Net',
        required=True
    )
    parser.add_argument(
        '--rnet',
        type=str,
        help='The path to the .hdf5 file containing the weights of R-Net',
        required=True
    )
    return parser.parse_args()

def main():

    rng = default_rng()

    # Get arguments
    args = get_arguments()

    # Create saving folder
    if not path.exists(args.output_directory):
        os.makedirs(args.output_directory)

    # Load P-Net
    # pnet = pnet_predict()
    pnet = pnet_predict_v2()
    pnet.load_weights(filepath=args.pnet, by_name=True)

    # Load R-Net
    rnet = rnet_predict()
    rnet.load_weights(filepath=args.rnet, by_name=True)

    # These are for storing the images
    pos_images = list()
    par_images = list()
    neg_images = list()

    # These are for storing the annotations of bounding box regression task
    pos_bboxes = list()
    par_bboxes = list()

    # Open the annotation file
    f = open(args.annotation_file)
    batch = 0
    for i in tqdm(range(12880)): # Because there are a total of 12880 images in the Wider Face dataset

        # Read the file line by line
        line = f.readline()

        # Obtain the number of faces in this image
        num_faces = int(f.readline())

        # Skip the image if it does not contain any faces
        if num_faces == 0:
            f.readline()
            continue

        # Obtain bounding boxes while taking into account invalid faces
        ground_truth = np.zeros((num_faces, 4), np.int32)
        invalid = []
        for j in range(num_faces):
            face = f.readline()
            if face[-7] == '1':
                invalid.append(j)
                continue
            ground_truth[j] = face[:-14].split(' ')

        # if i < 10000:
        #     continue

        # if rng.random() > 0.01:
        #     continue
        # count += 1

        # Remove invalid faces
        if len(invalid) > 0:
            ground_truth = np.delete(ground_truth, invalid, 0)
            if ground_truth.shape[0] == 0:
                continue
        # num_faces = ground_truth.shape[0]

        min_size = np.min(ground_truth[:, 2:])
        min_size = int(min_size * 0.6)
        min_size = max(min_size, 20)

        # Read the image
        img = cv2.imread(args.images_directory + '/' + line[:-1])[:, :, ::-1]
        # print(img.shape)

        # Get the original image size
        height, width = img.shape[:2]

        # Initialize the scale
        scale = 12 / min_size
        h = round(height * scale)
        w = round(width * scale)

        c_bboxes = list()
        confidence = list()

        # Start the loop
        while h >= 12 and w >= 12:

            # Resize the image
            resized_img = cv2.resize(img, (w, h)).reshape((1, h, w, 3))

            # Preprocess the image
            resized_img = np.add(resized_img, -127.5, dtype=np.float32) / 127.5

            # Pass the image to P-Net
            c_bboxes_i, confidence_i = pnet(resized_img)

            # If there are no faces, move on to next loop
            if c_bboxes_i.shape[0] == 0:
                scale = scale * 0.8
                h = round(height * scale)
                w = round(width * scale)
                continue

            c_bboxes.append(c_bboxes_i / scale)
            confidence.append(confidence_i)

            # Calculate the next scale
            scale = scale * 0.8
            h = round(height * scale)
            w = round(width * scale)

        # if i == 4155:
        #     print(c_bboxes)
        #     exit()

        # Apply non maximum suppression one last time
        # try:
        #     c_bboxes = tf.concat(c_bboxes, axis=0)
        # except:
        #     print(i)
        #     print(c_bboxes)
        #     print(len(c_bboxes))
        #     exit()
        if len(c_bboxes) == 0:
            continue
        c_bboxes = tf.concat(c_bboxes, axis=0)
        # print(c_bboxes.shape)
        confidence = tf.concat(confidence, axis=0)
        final_indices = tf.image.non_max_suppression(c_bboxes, confidence, confidence.shape[0], 0.7)
        c_bboxes = tf.gather(c_bboxes, final_indices)

        c_bboxes = square(c_bboxes)

        # print(c_bboxes.shape)
        # exit()

        # Round out the values
        c_bboxes = tf.cast(tf.round(c_bboxes), tf.int32)
        # print(type(c_bboxes))
        # print(c_bboxes.shape)
        # print(c_bboxes[0])
        # exit()

        c_bboxes = c_bboxes.numpy()[:, :3]
        c_bboxes[:, 2] = c_bboxes[:, 2] - c_bboxes[:, 0]
        # print(type(c_bboxes))
        # print(c_bboxes.shape)
        # print(c_bboxes[0])
        # exit()

        # print(type(ground_truth))
        # print(ground_truth.shape)
        # print(ground_truth[0])
        # # exit()

        ground_truth[:, :2] = ground_truth[:, :2][:, ::-1]
        ground_truth[:, 2:] = ground_truth[:, 2:][:, ::-1]

        # temp = ground_truth[:, 0].copy()
        # ground_truth[:, 0] = ground_truth[:, 1]
        # ground_truth[:, 1] = temp
        # temp = ground_truth[:, 2].copy()
        # ground_truth[:, 2] = ground_truth[:, 3]
        # ground_truth[:, 3] = temp

        # print(type(ground_truth))
        # print(ground_truth.shape)
        # print(ground_truth[0])
        # exit()

        # print(ground_truth.shape)
        # print(c_bboxes.shape)

        # Create R-Net input
        rnet_input = list()
        for bbox in c_bboxes:
            # print(bbox)
            # exit()
            rnet_input.append(crop_and_resize_v2(img, bbox, 24))
        rnet_input = np.asarray(rnet_input)
        # inspect(rnet_input)

        # Preprocess R-Net input
        rnet_input = np.add(rnet_input, -127.5, dtype=np.float32) / 127.5

        # Stage 2
        # print(type(c_bboxes))
        # print(rnet_input.shape)
        # print(c_bboxes.shape)
        c_bboxes = rnet([rnet_input, c_bboxes])
        # print(c_bboxes)
        # print(type(c_bboxes))
        # print(c_bboxes.shape)
        # print(c_bboxes[0])
        # show(img, c_bboxes.numpy())
        # if count == 15:
        #     exit()
        # continue
        # exit()

        c_bboxes = c_bboxes.numpy()[:, :3]
        c_bboxes[:, 2] = c_bboxes[:, 2] - c_bboxes[:, 0]

        # Calculate iou
        ious = find_iou_bulk(c_bboxes, ground_truth)

        # print(c_bboxes.shape[0])
        # print(np.count_nonzero(ious > 0.65))
        # exit()

        # h, w = img.shape[:2]
        # dpi = matplotlib.rcParams['figure.dpi']
        # figsize = w / dpi, h / dpi
        # figure, ax = plt.subplots(1, figsize=figsize)
        # ax.imshow(img)
        # for bbox in c_bboxes:
        #     bb = matplotlib.patches.Rectangle(xy=(bbox[1], bbox[0]), width=bbox[2], height=bbox[2], fill=False, edgecolor='b')
        #     ax.add_patch(bb)

        # Start creating data
        num_windows = c_bboxes.shape[0]
        max_iou_mask = np.argmax(ious, axis=1)
        max_ious = ious[np.arange(num_windows), max_iou_mask]

        # print(max_ious.shape)
        # exit()

        # Create positives
        pos_mask = max_ious > 0.65
        num_pos = np.count_nonzero(pos_mask)
        if num_pos > 0:
            pos_windows = c_bboxes[pos_mask]
            # num_pos = pos_windows.shape[0]
            pos_bboxes_i = create_bbr_annotation_v2_bulk(pos_windows, ground_truth[max_iou_mask[pos_mask]])
            pos_bboxes.extend(pos_bboxes_i)
            for bbox in pos_windows:
                # print(bbox)
                # bb = matplotlib.patches.Rectangle(xy=(bbox[1], bbox[0]), width=bbox[2], height=bbox[2], fill=False, edgecolor='g')
                # ax.add_patch(bb)
                pos_images.append(crop_and_resize_v2(img, bbox, 48))
        # else:
        #     num_pos = 0
        # print(ground_truth[max_iou_mask[max_ious > 0.65]])
        # print(create_bbr_annotation_v2_bulk(c_bboxes[max_ious > 0.65], ground_truth[max_iou_mask[max_ious > 0.65]]))
        # print(pos_bboxes)
        # print(pos_images[0].shape)
        # print(pos_images[0].shape)
        # exit()

        # Create part faces
        par_mask = np.logical_and(0.45 <= max_ious, max_ious <= 0.65)
        num_par = np.count_nonzero(par_mask)
        if num_par > 0:
            par_windows = c_bboxes[par_mask]
            par_bboxes_i = create_bbr_annotation_v2_bulk(par_windows, ground_truth[max_iou_mask[par_mask]])
            par_bboxes.extend(par_bboxes_i)
            for bbox in par_windows:
                # bb = matplotlib.patches.Rectangle(xy=(bbox[1], bbox[0]), width=bbox[2], height=bbox[2], fill=False, edgecolor='y')
                # ax.add_patch(bb)
                par_images.append(crop_and_resize_v2(img, bbox, 48))
        #     print(par_windows)
        #     print(ground_truth)
        # print(np.count_nonzero(par_mask))
        # print(par_mask)
        # exit()

        # Create negatives
        # print(num_windows)
        neg_mask = max_ious < 0.3
        num_neg = np.count_nonzero(neg_mask)
        # print(num_neg)
        if num_neg > 0:
            count = 0
            limit = 100
            size = []
            indices = rng.permutation(np.arange(num_windows)[neg_mask])
            # print(indices)
            for j in indices:
                if c_bboxes[j, 2] not in size:
                    size.append(c_bboxes[j, 2])
                    # print(c_bboxes[j])
                    count = count + 1
                    neg_images.append(crop_and_resize_v2(img, c_bboxes[j], 48))
                    # bbox = c_bboxes[j]
                    # bb = matplotlib.patches.Rectangle(xy=(bbox[1], bbox[0]), width=bbox[2], height=bbox[2], fill=False, edgecolor='r')
                    # ax.add_patch(bb)
                elif rng.random() < 0.1:
                    count = count + 1
                    neg_images.append(crop_and_resize_v2(img, c_bboxes[j], 48))
                    # bbox = c_bboxes[j]
                    # bb = matplotlib.patches.Rectangle(xy=(bbox[1], bbox[0]), width=bbox[2], height=bbox[2], fill=False, edgecolor='r')
                    # ax.add_patch(bb)
                if count == limit:
                    break

        # plt.axis('off')
        # plt.show()
        # if i == 5:
        #     exit()

        if (i + 1) % 1000 == 0:

            # Save the images
            pos_images = np.asarray(pos_images)
            par_images = np.asarray(par_images)
            neg_images = np.asarray(neg_images)
            np.save(args.output_directory + '/positive_images_' + str(batch) + '.npy', pos_images)
            np.save(args.output_directory + '/part_face_images_' + str(batch) + '.npy', par_images)
            np.save(args.output_directory + '/negative_images_' + str(batch) + '.npy', neg_images)
            pos_images = list()
            par_images = list()
            neg_images = list()

            # Save the bounding box annotations
            par_bboxes = np.asarray(par_bboxes)
            pos_bboxes = np.asarray(pos_bboxes)
            np.save(args.output_directory + '/part_faces_' + str(batch) + '.npy', par_bboxes)
            np.save(args.output_directory + '/positives_' + str(batch) + '.npy', pos_bboxes)
            pos_bboxes = list()
            par_bboxes = list()

            batch = batch + 1

    # Always close the file
    f.close()

    # Save the images
    pos_images = np.asarray(pos_images)
    par_images = np.asarray(par_images)
    neg_images = np.asarray(neg_images)
    np.save(args.output_directory + '/positive_images_' + str(batch) + '.npy', pos_images)
    np.save(args.output_directory + '/part_face_images_' + str(batch) + '.npy', par_images)
    np.save(args.output_directory + '/negative_images_' + str(batch) + '.npy', neg_images)

    # Save the bounding box annotations
    par_bboxes = np.asarray(par_bboxes)
    pos_bboxes = np.asarray(pos_bboxes)
    np.save(args.output_directory + '/part_faces_' + str(batch) + '.npy', par_bboxes)
    np.save(args.output_directory + '/positives_' + str(batch) + '.npy', pos_bboxes)

# def inspect(x):
#     print(type(x))
#     print(x.shape)
#     print(x.dtype)
#     exit()

def show(img, c_bboxes):
    h, w = img.shape[:2]
    print(h, w)
    dpi = matplotlib.rcParams['figure.dpi']
    figsize = w / dpi, h / dpi
    figure, ax = plt.subplots(1, figsize=figsize)
    ax.imshow(img)
    for bbox in c_bboxes:
        bb = matplotlib.patches.Rectangle(xy=(bbox[1], bbox[0]), width=bbox[3] - bbox[1], height=bbox[2] - bbox[0], fill=False, edgecolor='r')
        ax.add_patch(bb)
    plt.axis('off')
    plt.show()

def square(bboxes):
    """
    Turn the bounding boxes into squares.
    Args:
    - bboxes: (n, 4) tensor with each row encoded as [y1, x1, y2, x2]
    Return:
    - square_bboxes: (n, 4) tensor of squared bounding boxes
    """
    h = bboxes[:, 2] - bboxes[:, 0]
    w = bboxes[:, 3] - bboxes[:, 1]
    size = tf.math.maximum(h, w)
    x_margin = (size - w) / 2
    y_margin = (size - h) / 2
    y1 = bboxes[:, 0] - y_margin
    x1 = bboxes[:, 1] - x_margin
    y2 = bboxes[:, 2] + y_margin
    x2 = bboxes[:, 3] + x_margin
    square_bboxes = tf.stack([y1, x1, y2, x2], axis=1)
    return square_bboxes

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
"""
Created on Wed Dec  1 18:21:57 2021

@author: david.cheung1
"""
import pandas as pd
from io import StringIO
import glob
import math
from date import date
import logging
from tkinter import Tk
from tkinter.filedialog import askopenfilename
from datetime import datetime
import os.path

# function #################################################################################################################################################
def convert_dotout_or_excel_to_df(source_fileName: str , grid: int, header_pollutant: str) -> pd.DataFrame:
    if '.out' in source_fileName:
        f = open(f'{directory}/{grid}/{source_fileName}', 'r')
        data = f.read()[2:]
        df = pd.read_csv(StringIO(data))
        df.reset_index(inplace=True)
        df = df[list(df.columns[4:df.shape[1]-1])] # drop unwanted columns
        df['date'] = date_index.loc[:,('date')].astype('int64')
        # df['date'] = date_index['date'].astype('int64')
        df.set_index('date', inplace = True)

        # get header from reference excel
        for header_file in glob.glob(f'{directory}/{grid}/{grid}_{header_pollutant}*xlsx'):
            continue
        df1 = pd.read_excel(header_file)
        df =  df.set_axis(list(df1)[1:], axis=1, inplace=False)

    elif '.xlsx' in source_fileName:
        df = pd.read_excel(f'{directory}/{grid}/{source_fileName}')
        df = df[list(df.columns[1:])]
        df['date'] = date_index.loc[:,('date')].astype('int64')
        # df['date'] = date_index['date'].astype('int64')
        df.set_index('date', inplace = True)
    return(df)

def get_unique_pollutant_in_a_grid(grid: int) -> pd.array:
    totalPollutant = pd.unique(input1Table['AQO_Pollutant'][input1Table.Grid == grid])
    return totalPollutant

def get_background_CONC(pollutant:str, PATH_filename:str, Grid:int) -> pd.DataFrame:
    background_CONC = pd.read_csv(f'{directory}/{Grid}/{PATH_filename}', delim_whitespace=True)
    background_CONC = background_CONC.drop([0,1])
    background_CONC = background_CONC[background_CONC.columns[4:]]
    background_CONC = background_CONC.astype(str).astype(float)
    background_CONC.reset_index(inplace = True, drop = True)
    background_CONC['NOx'] = background_CONC['NO2'] + background_CONC['NO']
    background_CONC['date'] = date_index['date'].astype('int64')
    background_CONC.set_index('date', inplace = True)

    resultdf = background_CONC[f'{pollutant}'].to_frame()
    return resultdf

def jenkin_calculation(CONC: float) -> list:
    OX = float(inputPara['Value'][inputPara.Parameter == 'OX'])
    JK = float(inputPara['Value'][inputPara.Parameter == 'j/k'])
    NO2 = ((CONC + OX + JK) - math.sqrt((CONC + OX + JK)**2 - 4*CONC*OX))/2
    return NO2

def OLM_calculation(pollu_val_df:pd.DataFrame, NO_df:pd.DataFrame) -> pd.DataFrame:
    PATH_filename = pd.unique(pollu_val_df["Revised_PATH_Level1"])[0]
    Grid = pd.unique(pollu_val_df["Grid"])[0]
    pollutant = "O3"
    PATH_background_CONC_df = get_background_CONC(pollutant, PATH_filename, Grid).apply(lambda x: x*(46/48))
    PATH_background_CONC_df = PATH_background_CONC_df.rename(columns={'O3':'PATH_03'})
    NO_df = pd.concat([NO_df, PATH_background_CONC_df], axis=1)

    # case: lv1 + lv2 + lv3
    if str(pd.unique(pollu_val_df["ASR_Height_for_PATH_Level3"])[0]) != 'nan':
        PATH2_height = pd.unique(pollu_val_df["ASR_Height_for_PATH_Level2"])[0]
        PATH2_filename = pd.unique(pollu_val_df["PATH_Level2"])[0]
        PATH2_background_CONC_df = get_background_CONC(pollutant, PATH2_filename, Grid).apply(lambda x: x*(46/48))
        PATH2_background_CONC_df = PATH2_background_CONC_df.rename(columns={'O3':'PATH2_03'})
        NO_df = pd.concat([NO_df, PATH2_background_CONC_df], axis=1)

        PATH3_height = pd.unique(pollu_val_df["ASR_Height_for_PATH_Level3"])[0]
        PATH3_filename = pd.unique(pollu_val_df["PATH_Level_3"])[0]
        PATH3_background_CONC_df = get_background_CONC(pollutant, PATH3_filename, Grid).apply(lambda x: x*(46/48))
        PATH3_background_CONC_df = PATH3_background_CONC_df.rename(columns={'O3':'PATH3_03'})
        NO_df = pd.concat([NO_df, PATH3_background_CONC_df], axis=1)
        # logging.info('OLM calculation()  case2: lv1 + lv2 + lv3')
        for columns in NO_df:
            # sum up with lv1 PATH
            if float(columns.split('_')[-1]) < PATH2_height:
                NO_df[columns] = NO_df.apply(lambda row: min(row[columns], row["PATH_03"]), axis = 1)

            # sum up with lv2 PATH
            elif PATH2_height <= float(columns.split('_')[-1]) and float(columns.split('_')[-1]) < PATH3_height:
                NO_df[columns] = NO_df.apply(lambda row: min(row[columns], row["PATH2_03"]), axis = 1)

            # sum up with lv3 PATH
            elif PATH3_height <= float(columns.split('_')[-1]):
                NO_df[columns] = NO_df.apply(lambda row: min(row[columns], row["PATH3_03"]), axis = 1)

        NO_df = NO_df.drop(columns = ['PATH_03','PATH2_03','PATH3_03'])

    # case: lv1 + lv2
    elif str(pd.unique(pollu_val_df["ASR_Height_for_PATH_Level2"])[0]) != 'nan':
        PATH2_height = pd.unique(pollu_val_df["ASR_Height_for_PATH_Level2"])[0]
        PATH2_filename = pd.unique(pollu_val_df["PATH_Level2"])[0]
        PATH2_background_CONC_df = get_background_CONC(pollutant, PATH2_filename, Grid).apply(lambda x: x*(46/48))
        PATH2_background_CONC_df = PATH2_background_CONC_df.rename(columns={'O3': 'PATH2_03'})
        NO_df = pd.concat([NO_df, PATH2_background_CONC_df], axis=1)
        # logging.info('OLM calculation() case2: lv1 + lv2')
        for columns in NO_df:
            # sum up with lv1 PATH
            if float(columns.split('_')[-1]) < PATH2_height:
                NO_df[columns] = NO_df.apply(lambda row: min(row[columns], row["PATH_03"]), axis = 1)

            # sum up with lv2 PATH
            else:
                NO_df[columns] = NO_df.apply(lambda row: min(row[columns], row["PATH2_03"]), axis = 1)

        NO_df = NO_df.drop(columns = ['PATH_03','PATH2_03'])

    # case: lv1 only
    else:
        for columns in NO_df:
            NO_df[columns] = NO_df.apply(lambda row: min(row[columns], row["PATH_03"]), axis = 1)

        NO_df = NO_df.drop(columns = ['PATH_03'])

    return NO_df

def classify_pollutant_from_input1Table(pollu_val: str, jenkin: str) -> str:
    if pollu_val == 'NO2':
        if jenkin == 'Y':
            pollutant = 'NOx'
        else:
            pollutant = 'NO2'
    elif pollu_val == 'RSP':
        pollutant = 'RSP'
    elif pollu_val == 'FSP':
        pollutant = 'FSP'

    return pollutant

def classify_NOx_NO(OLM_df: pd.DataFrame)->pd.DataFrame:
    final_df = pd.DataFrame()
    NO_df = pd.DataFrame()
    NO2_df = pd.DataFrame()
    for i in OLM_df.index:
        source_fileName = str(OLM_df['Filename'].loc[i])
        grid = int(OLM_df['Grid'].loc[i])
        header_pollutant = str(OLM_df['AQO_Pollutant'].loc[i])
        ratio = float(OLM_df['NO/NO2_ratio'].loc[i])
        temp_df = convert_dotout_or_excel_to_df(source_fileName, grid, header_pollutant)
        # filter NOx, NO, NO2 in Source_Pollutant column
        if  str(OLM_df['Source_Pollutant'].loc[i]) == 'NOx':

            # get NO from NOx first
            df1 = temp_df.applymap(lambda x: x*ratio)
            if NO_df.shape[0] < 1:
                NO_df = df1
            else:
                NO_df = NO_df + df1
            # get NO2 from NOx
            df1 = temp_df.applymap(lambda x: x*(1 - ratio))
            if NO2_df.shape[0] < 1:
                NO2_df = df1
            else:
                NO2_df = NO2_df + df1

        elif str(OLM_df['Source_Pollutant'].loc[i]) == 'NO':
            if NO_df.shape[0] < 1:
                NO_df = temp_df
            else:
                NO_df = NO_df + temp_df

        else:

            if NO2_df.shape[0] < 1:
                NO2_df = temp_df
            else:
                NO2_df = NO2_df + temp_df

    if NO_df.shape[0] > 1:
        NO_df = OLM_calculation(OLM_df, NO_df)
        if final_df.shape[0] > 1:
            if NO2_df.shape[0] > 1:
                final_df = final_df + NO_df + NO2_df
            else:
                final_df = final_df + NO_df
        else:
            if NO2_df.shape[0] > 1:
                final_df = NO_df + NO2_df
            else:
                final_df = NO_df
    else:
        if final_df.shape[0] > 1:
            final_df = final_df + NO2_df
        else:
            final_df = NO2_df

    return final_df

def PATH_PATH2_PATH3(pollu_val_df: pd.DataFrame, process_df: pd.DataFrame, pollutant: str) -> pd.DataFrame:
    Grid = pd.unique(pollu_val_df["Grid"])[0]

    PATH_filename = pd.unique(pollu_val_df["Revised_PATH_Level1"])[0]
    PATH_background_CONC_df = get_background_CONC(pollutant, PATH_filename, Grid)
    PATH2_height = pd.unique(pollu_val_df["ASR_Height_for_PATH_Level2"])[0]
    PATH2_filename = pd.unique(pollu_val_df["PATH_Level2"])[0]
    PATH2_background_CONC_df = get_background_CONC(pollutant, PATH2_filename, Grid)
    PATH3_height = pd.unique(pollu_val_df["ASR_Height_for_PATH_Level3"])[0]
    PATH3_filename = pd.unique(pollu_val_df["PATH_Level_3"])[0]
    PATH3_background_CONC_df = get_background_CONC(pollutant, PATH3_filename, Grid)
    # logging.info('case: PATH + PATH2 + PATH3')
    for columns in process_df:
        # sum up with lv1 PATH
        if float(columns.split('_')[-1]) < PATH2_height:
            process_df[columns] = process_df[columns] + PATH_background_CONC_df[pollutant]

        # sum up with lv2 PATH
        elif PATH2_height <= float(columns.split('_')[-1]) and float(columns.split('_')[-1]) < PATH3_height:
            process_df[columns] = process_df[columns] + PATH2_background_CONC_df[pollutant]

        # sum up with lv3 PATH
        elif PATH3_height <= float(columns.split('_')[-1]):
            process_df[columns] = process_df[columns] + PATH3_background_CONC_df[pollutant]

    return process_df

def PATH_PATH2(pollu_val_df: pd.DataFrame, process_df: pd.DataFrame, pollutant: str) -> pd.DataFrame:
    Grid = pd.unique(pollu_val_df["Grid"])[0]

    PATH_filename = pd.unique(pollu_val_df["Revised_PATH_Level1"])[0]
    PATH_background_CONC_df = get_background_CONC(pollutant, PATH_filename, Grid)
    PATH2_height = pd.unique(pollu_val_df["ASR_Height_for_PATH_Level2"])[0]
    PATH2_filename = pd.unique(pollu_val_df["PATH_Level2"])[0]
    PATH2_background_CONC_df = get_background_CONC(pollutant, PATH2_filename, Grid)
    # logging.info('case: PATH + PATH2')
    for columns in process_df:
        # sum up with lv1 PATH
        if float(columns.split('_')[-1]) < PATH2_height:
            process_df[columns] = process_df[columns] + PATH_background_CONC_df[pollutant]

        # sum up with lv2 PATH
        else:
            process_df[columns] = process_df[columns] + PATH2_background_CONC_df[pollutant]

    return process_df

def jenkin_and_other_pollutant(pollu_val_df: pd.DataFrame) -> pd.DataFrame:
    AQO_Pollutant = str(pd.unique(pollu_val_df['AQO_Pollutant'])[0])
    # 1. fliter input dataframe to NO2 + jenkin if there is Jenkin
    if AQO_Pollutant == 'NO2':
        jenkin = "Y"
        pollu_val_df = pollu_val_df[(pollu_val_df.Jenkin_Method != "O")]
    else:
        jenkin = 'N'

    #  2. combine all the raw data file together as process_df
    pollutant = classify_pollutant_from_input1Table(AQO_Pollutant, jenkin)
    process_df = pd.DataFrame()
    for i in pollu_val_df.index:
        source_fileName = str(pollu_val_df['Filename'].loc[i])
        grid = int(pollu_val_df['Grid'].loc[i])
        header_pollutant = str(pollu_val_df['AQO_Pollutant'].loc[i])
        # logging.info(f'f(x)_jenkin_and_other_pollutant part 2: import data from {source_fileName}')
        if process_df.shape[0] < 1:
            process_df = convert_dotout_or_excel_to_df(source_fileName, grid, header_pollutant)

        else:
            process_df = process_df + convert_dotout_or_excel_to_df(source_fileName, grid, header_pollutant)

    # 3. get background concentration from PATH file + add background concentreation
    PATH_filename = pd.unique(pollu_val_df["Revised_PATH_Level1"])[0]
    Grid = pd.unique(pollu_val_df["Grid"])[0]
    PATH_background_CONC_df = get_background_CONC(pollutant, PATH_filename, Grid)

    # case: lv1 + lv2 + lv3
    if str(pd.unique(pollu_val_df["ASR_Height_for_PATH_Level3"])[0]) != 'nan':
        process_df = PATH_PATH2_PATH3(pollu_val_df, process_df, pollutant)
    # case: lv1 + lv2
    elif str(pd.unique(pollu_val_df["ASR_Height_for_PATH_Level2"])[0]) != 'nan':
        process_df = PATH_PATH2(pollu_val_df, process_df, pollutant)
    # case: lv1 only
    else:
        for columns in process_df:
            process_df[columns] = process_df[columns] + PATH_background_CONC_df[pollutant]

    # 4. apply Jenkin calculation if there is Jenkin
    if jenkin == 'Y':
        # logging.info('f(x)_jenkin_and_other_pollutant part 4: apply Jenkin calculation')
        process_df = process_df.applymap(lambda x : jenkin_calculation(x))

    return process_df

def OLM(pollu_val_df: pd.DataFrame) -> pd.DataFrame:
    # 0. create OLM df
    OLM_df = pollu_val_df[(pollu_val_df.AQO_Pollutant == 'NO2') & (pollu_val_df.Jenkin_Method != 'J')]
    OLM_df.reset_index(inplace = True,drop=True)
    # 1. classify into NO OLM, OLM1 only, OLM2 only, OLM3 only into 4 dataframe
    # NO OLM
    OLM0_df = OLM_df[(OLM_df.OLM1 != 'Y') & (OLM_df.OLM2 != 'Y') & (OLM_df.OLM3 != 'Y')]
    OLM0_df.reset_index(inplace = True,drop=True)
    # OLM1 only
    OLM1_df = OLM_df[(OLM_df.OLM1 == 'Y')]
    OLM1_df.reset_index(inplace = True,drop=True)
    # OLM2 only
    OLM2_df = OLM_df[(OLM_df.OLM2 == 'Y')]
    OLM2_df.reset_index(inplace = True,drop=True)
    # OLM3 only
    OLM3_df = OLM_df[(OLM_df.OLM3 == 'Y')]
    OLM3_df.reset_index(inplace = True,drop=True)

    # 2. append OLM df to dfs if OLM df has value
    lists = [OLM0_df,OLM1_df,OLM2_df,OLM3_df]
    dfs = []
    for df in lists:
        if df.shape[0] > 1:
            dfs.append(df)

    # 3. import raw data from source file in each dataframe
    result_df = pd.DataFrame()
    for df in dfs:
        if result_df.shape[0] < 1:
            result_df = classify_NOx_NO(df)
        else:
            result_df = result_df + classify_NOx_NO(df)
    # 4. sum up PATH background concentration
    PATH_filename = pd.unique(pollu_val_df["Revised_PATH_Level1"])[0]
    Grid = pd.unique(OLM_df["Grid"])[0]
    pollutant = "NO2"
    PATH_background_CONC_df = get_background_CONC(pollutant, PATH_filename, Grid)

    # case: lv1 + lv2 + lv3
    if str(pd.unique(OLM_df["ASR_Height_for_PATH_Level3"])[0]) != 'nan':
        result_df = PATH_PATH2_PATH3(OLM_df, result_df, pollutant)

    # case: lv1 + lv2
    elif str(pd.unique(OLM_df["ASR_Height_for_PATH_Level2"])[0]) != 'nan':
        result_df = PATH_PATH2(OLM_df, result_df, pollutant)
    # case: lv1 only
    else:
        for columns in result_df:
            result_df[columns] = result_df[columns] + PATH_background_CONC_df[pollutant]

    return result_df

def check_output_file_exist(export_filename:str) -> str:
    if os.path.isfile(f'{export_filename}'):
        mode = "a"
    else:
        mode = "w"

    return mode

def check_result_df(result_df:pd.DataFrame):
    if result_df.shape[0] < 1:
        logging.error(f'Group {i}, no result df reture in OLM part')

def export_result_df_to_excel(result_df:pd.DataFrame, export_filename:str, group_id:int, group_pollutant:str):
    check_result_df(result_df)
    mode = check_output_file_exist(export_filename)
    # export to output.xlsx
    with pd.ExcelWriter(f'{export_filename}',mode = mode) as writer:
                result_df.to_excel(writer, sheet_name=f'{group_id}_{group_pollutant}')

if __name__ == '__main__':
    # logging #################################################################################################################################################
    logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S')
    logging.info("Script start")

    # variable #################################################################################################################################################
    Tk().withdraw()
    logging.info("Please select Cumulative_Input excel in file dialog.")
    filename = askopenfilename()
    inputfile = filename
    directory = inputfile.rsplit('/',1)[0]
    date_index = pd.read_csv(StringIO(date))
    now = datetime.now()
    dt_string = now.strftime("%d%m%Y_%H%M")
    export_filename = f'{directory}/Output_{dt_string}.xlsx'

    input1Table = pd.read_excel(inputfile)
    input1Table.columns = input1Table.iloc[4]
    input1Table = input1Table.iloc[5:,0:18]
    input1Table.reset_index(inplace = True, drop = True)
    input1Table = input1Table.dropna(how='all')
    input1Table.columns = [c.replace(' ', '_') for c in input1Table.columns]

    inputPara = pd.read_excel(inputfile)
    inputPara = inputPara.iloc[5:,19:21]
    inputPara.reset_index(inplace = True, drop = True)
    inputPara = inputPara.rename(columns={'Unnamed: 19': 'Parameter', 'Unnamed: 20': 'Value'})

    totalId = pd.unique(input1Table['id'])

    # main script #################################################################################################################################################
    for i in totalId:
        pollu_val_df = input1Table[input1Table.id == i]

        # Step 1: based on group id (id column) to filter the input1Table into pollu_val_df
        # Step 2: input {pollu_val_df} to OLM() or jenkin_and_other_pollutant(), output would be result_df
        # Step 3: check the output {result_df} and export it to Output_(current_date)_(current_time).xlsx
        if "A" in pd.unique(pollu_val_df['Jenkin_Method']):
            # OLM + Jenkin
            logging.info(f'Group {i}, OLM + Jenkin')
            logging.info(f'Group {i}, OLM part start.')
            result_df = OLM(pollu_val_df)
            export_result_df_to_excel(result_df, export_filename, i, "NO2_OLM")
            logging.info(f'Group {i}, OLM part is done.')

            logging.info(f'Group {i}, Jenkin part start.')
            Jenkin_df = pollu_val_df[(pollu_val_df.AQO_Pollutant == 'NO2') & (pollu_val_df.Jenkin_Method != 'O')]
            result_df = jenkin_and_other_pollutant(Jenkin_df)
            export_result_df_to_excel(result_df, export_filename, i, "NO2_Jenkin")
            logging.info(f'Group {i}, Jenkin part is done.')

        else:
            # only OLM
            if "O" in pd.unique(pollu_val_df['Jenkin_Method']):
                logging.info(f'Group {i}, only OLM.')
                logging.info(f'Group {i}, OLM part start.')
                pollu_val_df = input1Table[(input1Table.AQO_Pollutant == 'NO2') & (input1Table.Jenkin_Method == 'O')]
                result_df = OLM(pollu_val_df)
                export_result_df_to_excel(result_df, export_filename, i, "NO2_OLM")
                logging.info(f'Group {i}, OLM part is done.')
            # only Jenkin
            elif "J" in pd.unique(pollu_val_df['Jenkin_Method']):
                logging.info(f'Group {i}, only Jenkin.')
                logging.info(f'Group {i}, Jenkin part start.')
                pollu_val_df = input1Table[(input1Table.AQO_Pollutant == 'NO2') & (input1Table.Jenkin_Method == 'J')]
                result_df = jenkin_and_other_pollutant(pollu_val_df)
                export_result_df_to_excel(result_df, export_filename, i, "NO2_Jenkin")
                logging.info(f'Group {i}, Jenkin part is done.')
            # other pollutant
            else:
                pollu_val = pd.unique(pollu_val_df['AQO_Pollutant'])[0]
                logging.info(f'Group {i}, other pollutant: {pollu_val}.')
                logging.info(f'Group {i}, {pollu_val} start calculation.')
                result_df = jenkin_and_other_pollutant(pollu_val_df)
                export_result_df_to_excel(result_df, export_filename, i, pollu_val)
                logging.info(f'Group {i}, {pollu_val} calculation is done.')

    logging.info("Script End")
    logging.info(f'Output file is in: {export_filename}')


#%% [markdown]
# ## Rock, Paper, Scissors Categorical Classifier with ConvNet

#%% [markdown]
# ### Loading required libraries
import os, signal
import zipfile
import tensorflow as tf
import matplotlib.pyplot as plt
import matplotlib.image as mpimage
import numpy as np
import random

#%% [markdown]
# ### Extract the zip file containing train data
local_zip = "./training-rps.zip"
zip_ref = zipfile.ZipFile(local_zip, "r")
zip_ref.extractall("./Training Data/Rock, Paper, Scissors")
zip_ref.close()

#%% [markdown]
# ### Extract the zip file containing validation data
local_zip = "./validation-rps.zip"
zip_ref = zipfile.ZipFile(local_zip, "r")
zip_ref.extractall("./Validation Data/Rock, Paper, Scissors")
zip_ref.close()

#%% [markdown]
# ### Defining training and validation images directories as path variables
train_rock_dir = os.path.join("./Training Data/Rock, Paper, Scissors/rock")
train_paper_dir = os.path.join("./Training Data/Rock, Paper, Scissors/paper")
train_scissors_dir = os.path.join("./Training Data/Rock, Paper, Scissors/scissors")
validation_rock_dir = os.path.join("./Validation Data/Rock, Paper, Scissors/rock")
validation_paper_dir = os.path.join("./Validation Data/Rock, Paper, Scissors/paper")
validation_scissors_dir = os.path.join(
    "./Validation Data/Rock, Paper, Scissors/scissors"
)

#%% [markdown]
# ### Let's see how the filenames look like
train_rock_names = os.listdir(train_rock_dir)
train_paper_names = os.listdir(train_paper_dir)
train_scissors_names = os.listdir(train_scissors_dir)
print(train_rock_names[:10])
print(train_paper_names[:10])
print(train_scissors_names[:10])
validation_rock_names = os.listdir(validation_rock_dir)
validation_paper_names = os.listdir(validation_paper_dir)
validation_scissors_names = os.listdir(validation_scissors_dir)
print(validation_rock_names[:10])
print(validation_paper_names[:10])
print(validation_scissors_names[:10])

#%% [markdown]
# ### Total number of training and validation images
print("Total training rock images: ", len(train_rock_names))
print("Total training paper images: ", len(train_paper_names))
print("Total training scissors images: ", len(train_scissors_names))
print("Total validation rock images: ", len(validation_rock_names))
print("Total validation paper images: ", len(validation_paper_names))
print("Total validation scissors images: ", len(validation_scissors_names))

#%% [markdown]
# ### Let's look at some of the pictures
nrows, ncols = 6, 4
pic_index = 0
random.seed(100)
rock_sample = random.sample(train_rock_names, 8)
rock_pics = [os.path.join(train_rock_dir, name) for name in rock_sample]
paper_sample = random.sample(train_paper_names, 8)
paper_pics = [os.path.join(train_paper_dir, name) for name in paper_sample]
scissors_sample = random.sample(train_scissors_names, 8)
scissors_pics = [os.path.join(train_scissors_dir, name) for name in scissors_sample]
fig = plt.gcf()
fig.set_size_inches(ncols * 2, nrows * 2)
for index, image_path in enumerate(rock_pics + paper_pics + scissors_pics):
    sp = plt.subplot(nrows, ncols, index + 1)
    sp.axis("Off")
    img = mpimage.imread(image_path)
    plt.imshow(img)
plt.show()

#%% [markdown]
# ### Structuring the ConvNet architecture
model = tf.keras.models.Sequential(
    [
        tf.keras.layers.Conv2D(
            64, (3, 3), activation="relu", input_shape=(300, 300, 3)
        ),
        tf.keras.layers.MaxPooling2D(2, 2),
        tf.keras.layers.Conv2D(64, (3, 3), activation="relu"),
        tf.keras.layers.MaxPooling2D(2, 2),
        tf.keras.layers.Conv2D(128, (3, 3), activation="relu"),
        tf.keras.layers.MaxPooling2D(2, 2),
        tf.keras.layers.Conv2D(128, (3, 3), activation="relu"),
        tf.keras.layers.MaxPooling2D(2, 2),
        tf.keras.layers.Flatten(),
        tf.keras.layers.Dropout(0.5),
        tf.keras.layers.Dense(512, activation="relu"),
        tf.keras.layers.Dense(3, activation="softmax"),
    ]
)

#%% [markdown]
# ### Get the model summary
model.summary()

#%% [markdown]
# ### Compiling the model
model.compile(
    optimizer=tf.keras.optimizers.RMSprop(lr=0.001),
    loss="categorical_crossentropy",
    metrics=["acc"],
)

#%% [markdown]
# ### Generating training and validation labeled data from images in directories
train_datagen = tf.keras.preprocessing.image.ImageDataGenerator(
    rescale=1 / 255,
    rotation_range=40,
    width_shift_range=0.2,
    height_shift_range=0.2,
    shear_range=0.2,
    zoom_range=0.2,
    horizontal_flip=True,
    fill_mode="nearest",
)
validation_datagen = tf.keras.preprocessing.image.ImageDataGenerator(rescale=1 / 255)
train_generator = train_datagen.flow_from_directory(
    "./Training Data/Rock, Paper, Scissors",
    target_size=(300, 300),
    class_mode="categorical",
)
validation_generator = validation_datagen.flow_from_directory(
    "./Validation Data/Rock, Paper, Scissors",
    target_size=(300, 300),
    class_mode="categorical",
)

#%% [markdown]
# ### Train the model
history = model.fit_generator(
    train_generator, epochs=25, verbose=1, validation_data=validation_generator
)

#%% [markdown]
# ### Plot training and validation accuracy
acc = history.history["acc"]
val_acc = history.history["val_acc"]
loss = history.history["loss"]
val_loss = history.history["val_loss"]
epochs = range(len(acc))
plt.plot(epochs, acc, "r", label="Training Accuracy")
plt.plot(epochs, val_acc, "b", label="Validation Accuracy")
plt.title("Training vs. Validation Accuracy")
plt.legend(loc=0)
plt.show()

#%% [markdown]
# ### Test classifier with real images
path = "./Validation Data/External Images/Rock, Paper, Scissors/scissors_test.jpg"
img = tf.keras.preprocessing.image.load_img(path, target_size=(300, 300))
x = tf.keras.preprocessing.image.img_to_array(img)
x = np.expand_dims(x, axis=0)
prediction = model.predict(x)
print(prediction)

#%% [markdown]
# ### Save the Model in a File and Cleanup memory
model.save("rps.h5")
os.kill(os.getpid(), signal.SIGKILL)


#!/usr/bin/python -u
#
# these functions exec psql instead of using a python adapter because
#  copy from/to don't work without root permissions but psql's \copy does.
#

from __future__ import absolute_import
import os
import sys
import re
import glob
import subprocess
import ConfigParser
import argparse

_UP = os.path.abspath( os.path.join( os.path.split( __file__ )[0], ".." ) )
sys.path.append( _UP )
import loader

#
# 1st row in file is column headers
# column order may not match database
#
# if table has upper-case letter, quote it
# ditto for column names
#
def _fromcsv( filename, dsn, schema, table, verbose = False ) :

# pgdb vs psycopg2
#
    if "database" in dsn.keys() : cmd = [ loader.PSQL, "-d", dsn["database"] ]
    else : cmd = [ loader.PSQL, "-d", dsn["dbname"] ]
    if "user" in dsn.keys() : cmd.extend( ["-U", dsn["user"]] )
    if "host" in dsn.keys() : cmd.extend( ["-h", dsn["host"]] )
    if "port" in dsn.keys() : cmd.extend( ["-p", dsn["port"]] )
    if not verbose : cmd.append( "-q" )

    infile = os.path.realpath( filename )
    if not os.path.exists( infile ) : raise IOError( "Not found: %s" % (infile,) )

# column names
#
    cols = []
    with open( infile, "rU" ) as f :
        l = f.readline()
        for col in l.split( "," ) :
            cols.append( col.strip().strip( "'\"" ) )

    if len( cols ) < 1 :
        sys.stderr.write( "no columns in %s\n" % (infile,) )
        return False

    colstr = ""
    for c in cols :
        if c.islower() : colstr += c
        else : colstr += '"' + c + '"'
        colstr += ","
    colstr = colstr[:-1]

    if (schema is None) or (str( schema ).strip() == "") :
        scam = ""
    else :
        scam = "%s." % (str( schema ).strip(),)

    if table.islower() :
        tbl = table
    else :
        tbl = '"%s"' % (table,)

    stmt = "\\copy %s%s (%s) from '%s' csv header" % (scam,tbl,colstr,infile,)

    cmd.extend( ["-c", stmt] )

    if verbose :
        sys.stderr.write( " ".join( j for j in cmd ) )
        sys.stderr.write( "\n" )

    p = subprocess.Popen( cmd, stdout = subprocess.PIPE, stderr = subprocess.PIPE )
    (out, err) = p.communicate()
    if p.returncode != 0 :
        sys.stderr.write( "ERR: psql returned %d\n" % (p.returncode,))
        sys.stderr.write( " ".join( j for j in cmd ) )
        sys.stderr.write( "\n" )
        sys.stderr.write( "** STDERR **\n" )
        for j in err.splitlines() :
            sys.stderr.write( j )
            sys.stderr.write( "\n" )
        sys.stderr.write( "** STDOUT **\n" )
        for j in out.splitlines() :
            sys.stderr.write( j )
            sys.stderr.write( "\n" )

        return False

    if verbose :
        sys.stderr.write( "** STDERR **\n" )
        for j in err.splitlines() :
            sys.stderr.write( j )
            sys.stderr.write( "\n" )
        sys.stderr.write( "** STDOUT **\n" )
        for j in out.splitlines() :
            sys.stderr.write( j )
            sys.stderr.write( "\n" )

    return True

# public wrapper
#
def fromcsv( dsn, filename, schema, table, verbose = False ) :
 return _fromcsv( filename, dsn, schema, table, verbose )

#TODO
#
# tables are <schema>.<table>.csv
#
#
def all_fromcsv( path, config, verbose = False ) :

    d = os.path.realpath( path )
    if not os.path.isdir( d ) : raise IOError( "Not a directory: %s" % (d,) )
    pat = re.compile( r"([^.]+)\.([^.]+)\.csv$" )
    errs = 0
    for i in glob.glob( os.path.join( d, "*.csv" ) ) :
        m = pat.search( os.path.split( i )[1] )
        if not m :
            sys.stderr.write( "%s does not match pattern, skipping\n" % (i,) )
            errs += 1
            continue

        if not _fromcsv( i, dsn = loader.dsn( config, "bmrbeverything" ),
                schema = m.group( 1 ), table = m.group( 2 ), verbose = verbose ) :
            errs += 1

    return (errs == 0)

#
#
#
if __name__ == "__main__" :

    ap = argparse.ArgumentParser( description = "load 'bmrbeverything' PostgreSQL database" )
    ap.add_argument( "-v", "--verbose", help = "print lots of messages to stdout", dest = "verbose",
        action = "store_true", default = False )

    ap.add_argument( "-c", "--config", help = "config file", dest = "conffile", required = True )
    ap.add_argument( "-d", "--dir", help = "directory with input files", dest = "indir", required = True )
    args = ap.parse_args()

    wd = os.path.realpath( args.indir )
    if not os.path.isdir( wd ) :
        sys.stderr.write( "Not a directory: %s\n", (wd,) )
        sys.exit( 1 )

    cp = ConfigParser.SafeConfigParser()
    f = os.path.realpath( args.conffile )
    cp.read( f )

    if not all_fromcsv( path = wd, config = cp, verbose = args.verbose ) :
        sys.exit( 1 )

#    if cp.has_option( "dictionary", "rouser" ) :
#        loader.add_ro_grants( dsn = loader.dsn( cp, "dictionary" ), schema = cp.get( "dictionary", "schema" ),
#            user = cp.get( "dictionary", "rouser" ),  verbose = args.verbose )
#
#
# eof


"""
deepdataspace.model.object

The object model.
"""

from typing import Dict
from typing import List
from typing import Literal
from typing import Optional
from typing import Union

from deepdataspace import constants
from deepdataspace.model._base import BaseModel
from deepdataspace.utils.file import create_file_url

class Object(BaseModel):
    """
    Objects are predictions, ground truths, or user annotations of an image.
    It is not stored in mongodb collections directly, but saved as nested documents in the image document.

    Attributes:
    -----------
    label_name: str
        The label name.
    label_type: str
        Is it a prediction? a GroundTruth? or a user annotation?, see :class:`deepdataspace.constants.LabelType`.
    label_id: str
        The label id.
    category_id: str
        The category id.
    category_name: str
        The category name.
    conf: float
        The confidence of the prediction.
    is_group: bool
        Is it a group of objects?
    bounding_box: dict
        The bounding box of the object, {"xmin": 0, "ymin": 0, "xmax": 0, "ymax": 0}.
    segmentation: str
        The segmentation of the object.
    alpha: str
        The alpha of the object.
    points: list
        The points of the object.
    lines: list
        The lines of the object.
    point_colors: list
        The point colors of the object.
    point_names: list
        The point names of the object.
    caption: str
        The caption of the object.
    confirm_type: int
        The image confirm type, 0 for unconfirmed, 1 for confirmed, 2 for rejected.
    compare_result: dict
        The compare result of the object, {"90": "FP", ..., "10": "OK"}.
    matched_det_idx: int
        The matched ground truth index, for prediction objects only.
    """

    @classmethod
    def get_collection(cls, *args, **kwargs):
        """
        Objects are stored directly inside the image document.
        """
        return None

    # the mandatory fields
    # every object must belong to a label set
    label_name: str
    label_type: str

    # the optional fields
    label_id: str = ""
    category_id: str = ""
    category_name: str = ""
    conf: Union[float, int] = 1.0
    is_group: Optional[bool] = False
    bounding_box: Optional[Dict[str, Union[float, int]]] = {}
    segmentation: Optional[str] = ""
    alpha: Optional[str] = ""
    points: Optional[List[Union[float, int]]] = []
    lines: Optional[List[int]] = []
    point_colors: Optional[List[int]] = []
    point_names: Optional[List[str]] = []
    caption: Optional[str] = ""
    confirm_type: Optional[int] = 0  # the image confirm type, 0 no confirm required, 1 gt may be fn, 2 pred may be fp
    compare_result: Optional[Dict[str, str]] = {}  # {"90": "FP", ..., "10": "OK"}
    matched_det_idx: Optional[int] = None  # The matched ground truth index, for prediction objects only.

    @staticmethod
    def _convert_file_path_to_url(file_uri: str):
        """
        Convert a local file path to a visible file url.
        """

        file_path = file_uri[7:]
        file_url = create_file_url(file_path=file_path,
                                   read_mode=constants.FileReadMode.Binary)
        return file_url

    def post_init(self):
        """
        Override the post_init method to convert the alpha file path to url.
        """

        if self.alpha and self.alpha.startswith("file://"):
            self.alpha = self._convert_file_path_to_url(self.alpha)


"""
Actuator that moves the lift platform inside the shaft to a given floor.

Interface is via service "move_to_floor(floor_number)", floor_number is 0:3
"""
import logging; logger = logging.getLogger("morse." + __name__)

import morse.core.actuator

from morse.core.services import service, async_service, interruptible
from morse.core import status
from morse.helpers.components import add_data, add_property

T=0.01**2 # max tollerence 1mm for position of platform

class Platform(morse.core.actuator.Actuator):
    _name = "Platform"
    _short_desc = "Move the elevator platform to given level"

    # define here the data fields required by your actuator
    # format is: field name, initial value, type, description
    #add_data('target_floor', 0, 'int', 'The floor to move to')
    add_data('current_floor', 0, 'int', 'The current floor. Read 999 if between floors.')

    def __init__(self, obj, parent=None):
        logger.info("%s initialization" % obj.name)
        # Call the constructor of the parent class
        super(self.__class__, self).__init__(obj, parent)

        # Do here actuator specific initializations

        self.local_data['current_floor'] = 0
        self.target_floor=0

        self.lift_cage= parent.bge_object.children['LiftCage']
        self.floor_height={-1:1.5, 0:4.5, 1:7.5, 2:10.5}

    @service
    def get_floor(self):
        """
        Return the current floor number
        """

        return self.local_data['current_floor']

    @interruptible
    @async_service
    def move_to_floor(self, floor):
        """ Move the platform to the given floor then return
        """
        self.target_floor = floor

    def default_action(self):
        """ Main loop of the actuator.

        Implements the component behaviour
        """

        # check if we have an on-going asynchronous task...
        if self.target_floor == self.local_data['current_floor']:
            self.completed(status.SUCCESS, self.local_data['current_floor'])

        # move the platform at given speed to target level.
        pos = self.lift_cage.localPosition
        distance =  self.floor_height[self.target_floor]  - pos[2]
        direction = -1 if distance < 0 else 1

        if distance**2 > T:
            vz = direction * 0.9 / self.frequency # 0.9m/s
        else:
            vz=0
            self.local_data['current_floor'] = self.target_floor

        self.lift_cage.localPosition[2]+=vz


import os
import re

import xlrd
from openpyxl import Workbook

def clean(text):
    if isinstance(text, (int, float)):
        return text
    if not text:
        return ''
    for c in ['\r\n', '\n\r', u'\n', u'\r', u'\t', u'\xa0', '\\']:
        text = text.replace(c, ' ')
    return re.sub(' +', ' ', text).strip()

class MergeExcelLeadsScript:
    file_headers = [
        "company name", "address", "city", "state", "zip", "country", "phone",
    ]

    def __init__(self):
        self.sheet_row_index = 1
        self.wb = Workbook()
        self.sheet = self.wb.active
        self.create_xls_write_headers()

        self.parse_excel_files()

        self.wb.save('All Leads.xlsx')

    def parse_excel_files(self):
        for fp in self.get_files_path():
            col_values = self.get_excel_rows(fp)
            raw = "\n".join(str(e).replace('.0', '') if e else "," for e in col_values)

            for r in raw.split('\n,'):
                rows = [e.strip() for e in r.split('\n') if clean(e)]

                item = {}
                item["company name"] = rows[0].strip()
                item["address"] = clean(' '.join(rows[1].split(',')[::-1]))
                item["city"] = clean(rows[1].split(',')[-1])
                item["country"] = clean(rows[2])
                item["state"] = clean(rows[3])
                item["zip"] = int(rows[4]) if rows[4:] and clean(rows[4]).isdigit() else ''
                item["phone"] = rows[5] if rows[5:] and len(rows[5]) > 7 else ''

                self.write_xls(item)

            a = 0

    def get_excel_rows(self, file_path):
        wb = xlrd.open_workbook(file_path)
        sheet = wb.sheet_by_index(0)
        # return [sheet.cell_value(i, 0) for i in range(1, sheet.nrows)]
        return sheet.col_values(0)

    def get_files_path(self):
        return [f"data/{file}" for file in os.listdir('data') if file.endswith(".xlsx")]

    def create_xls_write_headers(self):
        for index, value in enumerate(self.file_headers):
            self.sheet.cell(row=self.sheet_row_index, column=index + 1).value = value
        self.sheet_row_index += 1

    def write_xls(self, item):
        for idx, key in enumerate(self.file_headers):
            self.sheet.cell(row=self.sheet_row_index, column=idx + 1).value = item[key]

        self.sheet_row_index += 1
        print(item)

if __name__ == "__main__":
    MergeExcelLeadsScript()


#!/usr/bin/env python3

import argparse
import os
from pprint import pprint
import sys

from script_utils import print_error, print_section, print_tx_address

from clove.utils.search import get_network_by_symbol

sys.path.append(os.path.dirname(os.path.realpath(__file__)))

if __name__ == '__main__':

    parser = argparse.ArgumentParser(description="Generate and publish atomic swap initial transaction.")
    parser.add_argument('-n', '--network', help='Network for creating transaction in', type=str, required=True)
    parser.add_argument('-s', '--sender', help='Sender address', type=str, required=True)
    parser.add_argument('-r', '--recipient', help='Recipient address', type=str, required=True)
    parser.add_argument('-a', '--amount', help='Transaction amount', type=float, required=True)
    parser.add_argument('-p', '--private-key', help='Private key', type=str, required=True)
    args = parser.parse_args()

    network = get_network_by_symbol(args.network)

    print_section('Creating transaction for', args.network)
    print_section('Sender address:\t', args.sender)
    print_section('Recipient address:\t', args.recipient)
    print_section('Transaction amount:\t', args.amount)

    wallet = network.get_wallet(private_key=args.private_key)
    utxo = network.get_utxo(args.sender, args.amount)
    if not utxo:
        print_error('UTXO not found')
        exit(1)

    print_section(f'Found {len(utxo)} UTXO\'s')
    pprint(utxo)
    transaction = network.atomic_swap(args.sender, args.recipient, args.amount, utxo)

    print_section('Adding fee and signing')
    transaction.add_fee_and_sign(wallet)

    print_section('Transaction ready to be published')
    details = transaction.show_details()
    pprint(details)

    publish = input('Do you want to publish this transaction (y/n): ')
    if publish != 'y':
        print_section('Bye!')
        exit()

    print_section('Publishing transaction')
    if not transaction.publish():
        print_error('Something went wrong, transaction was NOT published.')
        exit(1)
    print_section('Transaction published!')
    print_tx_address(args.network, details["transaction_address"])


import pandas as pd
import dateparser

try:
    # change
    user_reviews = pd.read_csv('./DataFiles/reviews.csv')
    review_num_of_rows = user_reviews.shape[0]
except:
    print('Error while handling user_reviews file.')

try:
    # change
    apps = pd.read_csv('./DataFiles/apps.csv', parse_dates=["Last Updated"], date_parser=lambda x: dateparser.parse(x))
    apps_num_of_rows = apps.shape[0]
except:
    print('Error while handling apps file.')

try:
    if __name__ == '__main__':
        print('user_reviews table:')
        print('Number of rows: ', review_num_of_rows)
        print('Columns names: ', list(user_reviews.columns))
        print('\napps table:')
        print('Number of rows: ', apps_num_of_rows)
        print('Columns names: ', list(apps.columns))

        # number of categories
        apps_categories_count = apps['Category'].drop_duplicates().size
        print('\nTotal count of apps categories: {}'.format(apps_categories_count))

        # heaviest app
        max_size = apps[apps['Size'].str.contains('M$', na=False)]['Size'].max()
        heaviest_app = apps[apps['Size'] == max_size]['App'].drop_duplicates().values
        print('\nThe heaviest apps: {}'.format(heaviest_app))

        # most installations
        max_installs = apps['Installs'].max()
        most_installed_app = apps[apps['Installs'] == max_installs]['App'].drop_duplicates().values
        print('\nThe apps with most installations: {}'.format(most_installed_app))

        # most updated
        max_update_date = apps['Last Updated'].max()
        updated_app = apps[apps['Last Updated'] == max_update_date]['App'].drop_duplicates().values
        print('\nThe most updated apps: {}'.format(updated_app))

        # most popular app genres
        most_popular_app = apps[apps['Installs'] == apps['Installs'].max()][
            ['App', 'Rating', 'Genres']].drop_duplicates()
        most_popular_app_max_rating = most_popular_app['Rating'].max()
        most_popular_app_genre = most_popular_app[most_popular_app['Rating'] == most_popular_app_max_rating][
            'Genres'].drop_duplicates()
        print('\nThe most popular app\'s genres: {}'.format(most_popular_app_genre.values))
except:
    print('Error while handling apps file.')

# The tables has no PK or Fk, because they don't have a unique values. we understand that the column 'App' in user_reviwe
# should be a Fk, but in reality there are apps in user_review which does not exist in apps table.


import pandas as pd
import numpy as np

from sklearn import model_selection
from sklearn import linear_model
from sklearn import naive_bayes
from sklearn import svm
from sklearn import metrics
from sklearn import feature_selection

COLUMNS_NAME = ['age', 'workclass', 'fnlwgt', 'education', 'education-num',
                'marital-status', 'occupation', 'relationship', 'race', 'gender',
                'capital-gain', 'capital-loss', 'hours-per-week', 'native-country', 'income']

DATA_PATH = ('../uci_data/adult.data.txt',
             '../uci_data/adult.test.txt')

def preprocess(data):
    # remove row with missing value
    data = data[data["workclass"] != "?"]
    data = data[data["occupation"] != "?"]
    data = data[data["native-country"] != "?"]

    data.replace(['Married-civ-spouse', 'Divorced', 'Never-married',
                  'Separated', 'Widowed', 'Married-spouse-absent', 'Married-AF-spouse'],
                 ['married', 'not married', 'not married',
                  'not married', 'not married', 'married', 'married'], inplace=True)

    category_col =['race','marital-status', 'gender', 'income', 'relationship',
                   'native-country', 'workclass', 'occupation', 'education']
    for col in category_col:
        _, c = np.unique(data[col], return_inverse=True)
        data[col] = c

    return data

def load_data():
    train_data = pd.read_csv(DATA_PATH[0], header=None, names=COLUMNS_NAME)
    test_data = pd.read_csv(DATA_PATH[1], header=None, names=COLUMNS_NAME)

    # the first row is invalid
    test_data = test_data.drop([0])

    train_data = preprocess(train_data)
    test_data = preprocess(test_data)

    return (train_data, test_data)

def cal_scores(y_pred, y_true, name):
    acc = metrics.accuracy_score(y_test, y_pred)
    auc = metrics.roc_auc_score(y_test, y_pred)
    logloss = metrics.log_loss(y_test, y_pred)
    print('{name} \nacc: {0} \nauc:{1} \nlogloss:{2}\n========='.format(acc, auc, logloss, name=name))

def naive_bayes_model(x_train, y_train, x_test, y_test):
    model = naive_bayes.GaussianNB()
    model.fit(x_train, y_train)
    y_pred = model.predict(x_test)

    cal_scores(y_pred, y_test, 'GaussianNB')

def logistic_regress_model(x_train, y_train, x_test, y_test):
    model = linear_model.LogisticRegressionCV()
    model.fit(x_train, y_train)
    y_pred = model.predict(x_test)

    cal_scores(y_pred, y_test, 'Logistic')

def svm_model(x_train, y_train, x_test, y_test):
    model = svm.SVC()
    model.fit(x_train, y_train)
    y_pred = model.predict(x_test)

    cal_scores(y_pred, y_test, 'SVC')

if __name__ == '__main__':
    train_data, test_data = load_data()

    feature_cols = ['marital-status', 'education-num', 'relationship', 'age']
    y_train = train_data['income']
    x_train = train_data[feature_cols]
    y_test = test_data['income']
    x_test = test_data[feature_cols]

    naive_bayes_model(x_train, y_train, x_test, y_test)
    logistic_regress_model(x_train, y_train, x_test, y_test)
    svm_model(x_train, y_train, x_test, y_test)


import dataclasses

import numpy as np
import pandas as pd
import pytest

from . import close
from .conftest import TEST
from .. import analysis, factory
from ..measures import Accuracy

@pytest.fixture(scope="module")
def d_expert(fixt_data_factory):  # see conftest.py
    d = fixt_data_factory.make(data_src="test_expert")
    return d

@pytest.fixture(scope="module")
def d_others(fixt_data_factory):
    d_others_ = {}
    for i in range(2):
        data_src = f"test_crowd_{i}"
        d_others_[data_src] = fixt_data_factory.make(data_src=data_src)
    # print([d.data_src for d in d_others_.values()])
    return d_others_

@pytest.fixture(scope="module")
def expert_consensuses_n_parameters(d_expert):
    model_ = factory.Factory.make("DawidSkene")
    expert_consensuses, expert_parameters = model_.fit_and_compute_consensuses_from_data(d_expert, TEST.QUESTIONS)
    return expert_consensuses, expert_parameters

@pytest.fixture(scope="module")
def parameters_others(expert_consensuses_n_parameters, d_others):
    model_ = factory.Factory.make("DawidSkene")
    expert_consensuses, _ = expert_consensuses_n_parameters
    parameters_others_ = analysis.compute_crossed(model_, d_others, expert_consensuses)
    return parameters_others_

def test_compute_crossed(expert_consensuses_n_parameters, parameters_others):
    _, expert_parameters = expert_consensuses_n_parameters
    for question, dgp in expert_parameters.items():
        expert_dgp = dataclasses.asdict(dgp)
        for param_name, expert_param_val in expert_dgp.items():
            for other in parameters_others:
                other_dgp = dataclasses.asdict(parameters_others[other][question])
                assert np.array_equal(expert_param_val, other_dgp[param_name])

def test_prospective_analysis(d_expert, expert_consensuses_n_parameters, parameters_others):
    _, expert_parameters = expert_consensuses_n_parameters
    measures = {Accuracy.name: Accuracy}
    repeats = 2
    question = TEST.QUESTIONS[0]
    model_ = factory.Factory.make("DawidSkene")
    ds = factory.Factory.make("DawidSkene")
    mv = factory.Factory.make("MajorityVoting")
    models = {"DS": ds, "MV": mv}
    options = {"n_tasks": [10, 100],
               "n_annotations_per_task": [3, 9]}
    scenarios = ds.DataGenerationParameters.product_from_dict(options)
    df = analysis.prospective_analysis(question, d_expert.data_src, expert_parameters, parameters_others,
                                       generative_model=model_, models=models, measures=measures, dgps=scenarios,
                                       repeats=repeats)
    # print(df)
    assert isinstance(df, pd.DataFrame)
    assert df.shape[0] == repeats * len(models) * len(options["n_tasks"]) * \
           len(options["n_annotations_per_task"]) * (len(parameters_others.keys()) + 1)  # +1 for experts

def test_compare_data_to_consensus(d_expert, d_others, expert_consensuses_n_parameters):
    expert_consensuses, _ = expert_consensuses_n_parameters
    for d_other in d_others.values():
        for question in TEST.QUESTIONS:
            df = analysis.compare_data_to_consensus(d_expert, d_other, expert_consensuses, question,
                                                    add_total_cols=True)
            # assert df is DataFrame
            assert isinstance(df, pd.DataFrame)
            # #Rows == #Labels
            assert df.shape[0] == d_other.n_labels(question)
            # #Columns == #Classes
            assert (df.shape[1] - 1 == len(d_other.get_classes(question)))
            # Total column == Sum(other columns)
            assert np.allclose(df.iloc[:, -1].to_numpy(), df.iloc[:, :-1].sum(axis=1).to_numpy())

def test_gen_confusion_matrix(d_expert, expert_consensuses_n_parameters):
    expert_consensuses, _ = expert_consensuses_n_parameters
    for question in TEST.QUESTIONS:
        df = analysis.gen_confusion_matrix(expert_consensuses[question], expert_consensuses[question],
                                           d_expert, question)
        # assert df is DataFrame
        assert isinstance(df, pd.DataFrame)
        # assert all categories are in the index and in the columns
        categories = d_expert.get_categories()[question].categories.tolist()
        assert df.index.to_list() == categories
        assert df.columns.to_list() == categories
        # assert dataframe total sum = # of tasks
        assert np.nansum(df.to_numpy()) == len(np.unique(d_expert.get_tasks(question)))
        # TODO (OM, 20210716): assert the values of rows?


import socket
import os
import struct
from ctypes import *
import chardet
import re

#ip header
#----------------------32bit------------------------------------
#		8		|		8		|		8		|		8		|
#---------------------------------------------------------------
#版本4|包头长4	|	服务类型8	|			总长16				|
#---------------------------------------------------------------
#			标识16				|标志3|			偏移			|
#---------------------------------------------------------------
#		TTL		|		协议	|			checksum			|
#---------------------------------------------------------------
#						source_addr								|
#---------------------------------------------------------------
#						des_addr								|
#---------------------------------------------------------------
#					Option	and padding							|
#---------------------------------------------------------------

# host to listen on
host = "169.254.215.140"

#inherent Structure or Union
class IP(Structure):
    _fields_ = [
            ("ihl", c_ubyte, 4),
            ("version", c_ubyte, 4),
            ("tos", c_ubyte),
            ("len", c_ushort),
            ("id", c_ushort),
            ("offset", c_ushort),
            ("ttl", c_ubyte),
            ("protocol_num", c_ubyte),
            ("sum", c_ushort),
            ("src", c_ulong),
            ("dst", c_ulong)
        ]
    #用来创建实例
    def __new__(self, socket_buffer=None):
		#返回一个ctypes实例,此处是IP类的实例(继承)
        return self.from_buffer_copy(socket_buffer)
    #初始化实例
    def __init__(self, socket_buffer=None):
        # map protocol constants to their names
        self.protocol_map = {1:"ICMP", 6:"TCP", 17:"UDP"}
        # human readable IP addresses,<L unsign long little-endian
        self.src_address = socket.inet_ntoa(struct.pack("<L",self.src))
        self.dst_address = socket.inet_ntoa(struct.pack("<L",self.dst))
        # human readable protocol
        try:
            self.protocol = self.protocol_map[self.protocol_num]
        except:
            self.protocol = str(self.protocol_num)
        #2F00 -> 002F,总长
        self.length = socket.ntohs(self.len)
        #ip头长
        self.header_length = self.ihl*4;
        #identifier标志
        self.label = socket.ntohs(self.id)
        #offset
        self.off_set = socket.ntohs(self.offset)&0b1111111111111
        #flag标识 3bit 第二位表示是否分片，第三位表示若分片后面还有没有分片
        self.flag = socket.ntohs(self.offset)>>13

# create a raw socket and bind it to the public interface
if os.name == "nt":
    socket_protocol = socket.IPPROTO_IP
else:
    socket_protocol = socket.IPPROTO_ICMP

sniffer = socket.socket(socket.AF_INET, socket.SOCK_RAW, socket_protocol)
sniffer.bind((host, 60001))
# we want the IP headers included in the capture.
# On the other hand,a TCP/IP service provider that supports SOCK_RAW should also support IP_HDRINCL.
sniffer.setsockopt(socket.IPPROTO_IP, socket.IP_HDRINCL, 1)

#recv all
if os.name == "nt":
    sniffer.ioctl(socket.SIO_RCVALL, socket.RCVALL_ON)
try:
    while True:
        # read in a single packet,[data,src]
        raw_buffer = sniffer.recvfrom(65535)
        #数据
        raw_buffer = raw_buffer[0]
        #IP header
        ip_split = IP(raw_buffer[0:20])
        print("Protocol: %s %s -> %s" % (ip_split.protocol, ip_split.src_address, ip_split.dst_address))
        print("包头:%d" % ip_split.header_length)
        print("数据长度:%d字节" % (ip_split.length-40))
        print("偏移:%d字节" % (ip_split.off_set))
        print("标识:%s" % ip_split.label)
        print("标志位:%s" % bin(ip_split.flag))

		#extra data
        if(ip_split.length != 40):
            enc = chardet.detect(raw_buffer[40:])['encoding']
            if(enc):
                try:
                    print("数据:%s" % raw_buffer[40:].decode(enc))
                except:
                    print("Decode Error!!!")
            else:
                print("Unknow Encoding!")
        else:
            print("No Data!!!")
        print('-------------------------------')
except KeyboardInterrupt:
	if os.name == "nt":
		sniffer.ioctl(socket.SIO_RCVALL, socket.RCVALL_OFF)