import logging
import os
import uuid
from io import BytesIO
from typing import Sequence
from zipfile import ZIP_DEFLATED, ZipFile

import anndata as ad
import dramatiq
from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile
from fastapi.responses import ORJSONResponse
from matplotlib.colors import rgb2hex
from sqlalchemy.orm import Session
from starlette.responses import StreamingResponse
from starlette.status import HTTP_404_NOT_FOUND

from histocat.api.db import get_db
from histocat.api.security import get_active_member
from histocat.config import config
from histocat.core.dataset import service as dataset_service
from histocat.core.dataset.dto import DatasetDto, DatasetUpdateDto
from histocat.core.image import get_qualitative_colors
from histocat.core.member.models import MemberModel
from histocat.core.utils import stream_bytes

logger = logging.getLogger(__name__)
router = APIRouter()

@router.get("/groups/{group_id}/projects/{project_id}/datasets", response_model=Sequence[DatasetDto])
def get_project_datasets(
    group_id: int,
    project_id: int,
    db: Session = Depends(get_db),
    member: MemberModel = Depends(get_active_member),
):
    """Retrieve project's datasets"""
    items = dataset_service.get_project_datasets(db, project_id=project_id)
    return items

@router.patch("/groups/{group_id}/datasets/{dataset_id}", response_model=DatasetDto)
def update(
    group_id: int,
    dataset_id: int,
    params: DatasetUpdateDto,
    member: MemberModel = Depends(get_active_member),
    db: Session = Depends(get_db),
):
    """Update dataset"""
    item = dataset_service.get(db, id=dataset_id)
    if not item:
        raise HTTPException(status_code=HTTP_404_NOT_FOUND, detail=f"Dataset id:{dataset_id} not found")
    item = dataset_service.update(db, item=item, params=params)
    return item

@router.get("/groups/{group_id}/datasets/{dataset_id}/centroids")
def get_centroids(
    group_id: int,
    dataset_id: int,
    member: MemberModel = Depends(get_active_member),
    db: Session = Depends(get_db),
):
    """Get dataset cell centroids"""
    dataset = dataset_service.get(db, id=dataset_id)
    if not dataset:
        raise HTTPException(status_code=HTTP_404_NOT_FOUND, detail=f"Dataset id:{dataset_id} not found")

    adata = ad.read_h5ad(dataset.cell_file_location())
    mappable = get_qualitative_colors(vmin=adata.obs["AcquisitionId"].min(), vmax=adata.obs["AcquisitionId"].max())
    colors = [rgb2hex(c) for c in mappable.to_rgba(adata.obs["AcquisitionId"])]
    output = {
        "acquisitionIds": adata.obs["AcquisitionId"].tolist(),
        "cellIds": adata.obs["CellId"].tolist(),
        "objectNumbers": adata.obs["ObjectNumber"].tolist(),
        "x": adata.obs["CentroidX"].round(2).tolist(),
        "y": adata.obs["CentroidY"].round(2).tolist(),
        "colors": colors,
    }
    return ORJSONResponse(output)

@router.get("/groups/{group_id}/datasets/{dataset_id}", response_model=DatasetDto)
def get_by_id(
    group_id: int,
    dataset_id: int,
    member: MemberModel = Depends(get_active_member),
    db: Session = Depends(get_db),
):
    """Get dataset by id"""
    item = dataset_service.get(db, id=dataset_id)
    if not item:
        raise HTTPException(status_code=HTTP_404_NOT_FOUND, detail=f"Dataset id:{dataset_id} not found")
    return item

@router.delete("/groups/{group_id}/datasets/{dataset_id}", response_model=DatasetDto)
def delete_by_id(
    group_id: int,
    dataset_id: int,
    member: MemberModel = Depends(get_active_member),
    db: Session = Depends(get_db),
):
    """Delete a specific dataset by id"""
    item = dataset_service.remove(db, id=dataset_id)
    if not item:
        raise HTTPException(status_code=HTTP_404_NOT_FOUND, detail=f"Dataset id:{dataset_id} not found")
    return item

@router.get("/datasets/{dataset_id}/download")
async def download_by_id(dataset_id: int, db: Session = Depends(get_db)):
    """Download dataset by id"""
    item = dataset_service.get(db, id=dataset_id)
    if not item:
        raise HTTPException(status_code=HTTP_404_NOT_FOUND, detail=f"Dataset id:{dataset_id} not found")

    file_name = f"{item.name}.zip"
    abs_src = os.path.abspath(item.location)
    buffer = BytesIO()
    with ZipFile(buffer, "w", ZIP_DEFLATED) as zip:
        for folderName, _, filenames in os.walk(item.location):
            for filename in filenames:
                absname = os.path.abspath(os.path.join(folderName, filename))
                arcname = absname[len(abs_src) + 1 :]
                zip.write(absname, arcname)

    headers = {"Content-Disposition": f'attachment; filename="{file_name}"'}
    return StreamingResponse(stream_bytes(buffer.getvalue()), media_type="application/zip", headers=headers)

@router.post("/groups/{group_id}/projects/{project_id}/datasets/upload")
def upload_dataset(
    group_id: int,
    project_id: int,
    type: str = Form(None),
    masks_folder: str = Form(None),
    regionprops_folder: str = Form(None),
    intensities_folder: str = Form(None),
    file: UploadFile = File(None),
    member: MemberModel = Depends(get_active_member),
    db: Session = Depends(get_db),
):
    path = os.path.join(config.INBOX_DIRECTORY, str(uuid.uuid4()))
    if not os.path.exists(path):
        os.makedirs(path)
    uri = os.path.join(path, file.filename)
    with open(uri, "wb") as f:
        f.write(file.file.read())

    broker = dramatiq.get_broker()
    message = dramatiq.Message(
        actor_name="import_dataset",
        queue_name="import",
        args=(),
        kwargs={
            "type": type,
            "masks_folder": masks_folder,
            "regionprops_folder": regionprops_folder,
            "intensities_folder": intensities_folder,
            "uri": uri,
            "project_id": project_id,
        },
        options={},
    )
    broker.enqueue(message)
    return {"uri": uri}


import logging
import sys
import torch
import yaml

from tagging_trainer import TaggingTrainer

from forte.common.configuration import Config

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)

if __name__ == "__main__":
    task = sys.argv[1]
    assert task in ["ner", "pos"], "Not supported nlp task type: {}".format(
        task
    )

    extractor_configs = yaml.safe_load(
        open("configs/config_extractors.yml", "r")
    )

    # Configure output extractor based on the task, see
    #   BioSeqTaggingExtractor for more information.
    output_configs = extractor_configs["feature_scheme"]["output_tag"][
        "extractor"
    ]["config"]
    if task == "ner":
        output_configs["entry_type"] = "ft.onto.base_ontology.EntityMention"
        output_configs["attribute"] = "ner_type"
        output_configs["tagging_unit"] = "ft.onto.base_ontology.Token"
    elif task == "pos":
        output_configs["entry_type"] = "ft.onto.base_ontology.Token"
        output_configs["attribute"] = "pos"

    config = {
        "config_data": Config(
            {},
            default_hparams=yaml.safe_load(
                open("configs/config_data.yml", "r")
            ),
        ),
        "config_model": Config(
            {},
            default_hparams=yaml.safe_load(
                open("configs/config_model.yml", "r")
            ),
        ),
        "config_extractors": extractor_configs,
        "device": torch.device("cuda")
        if torch.cuda.is_available()
        else torch.device("cpu"),
    }

    trainer: TaggingTrainer = TaggingTrainer(task_type=task, **config)
    trainer.run()

    # Save training state to disk
    trainer.save(config["config_data"].train_state_path)
    torch.save(trainer.model, "model.pt")


import pathlib
import pandas as pd
from model import dez

datadir = pathlib.Path(__file__).parents[0].joinpath('data')

def test_populate_solution_land_distribution():
    expected = pd.read_csv(datadir.joinpath('lbt_ocean_dist.csv'), index_col=0)
    de = dez.DEZ('Limiting bottom trawling')

    # We freeze applicable zones as solution_dez_matrix.csv is likely to change
    de.applicable_zones = ['DEZ1: Epipelagic, EEZ', 'DEZ2: Epipelagic, ABNJ']
    de._populate_world_ocean_allocation()
    de._populate_solution_ocean_distribution()

    pd.testing.assert_frame_equal(de.get_ocean_distribution(), expected, check_dtype=False)


import asyncio
import logging
from random import randint

from src.streaming import Consumer, Producer, Topic

LOGGER = logging.getLogger(__name__)

GROUP_ID = "WEBSITE_NER"

async def amain():
    """Consume website changes, produce NER results."""
    consumer = await Consumer.create(Topic.CHANGE.value, group_id=GROUP_ID)
    producer = await Producer.create()
    async for message in consumer:
        data = message.value
        LOGGER.info("processing event %s %s", data["id"], data["domain"])
        await producer.send_and_wait(Topic.NER.value, calculate_ner(data))

def calculate_ner(data: dict) -> dict:
    return {
        "domain": data["domain"],
        "id": data["id"],
        # some fancy NER detection results
        "ner": {
            "persons": randint(0, 10),
            "locations": randint(0, 10),
            "brands": randint(0, 10),
        },
    }

if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s [%(module)s:%(lineno)s] %(message)s')
    asyncio.run(amain())


#!/usr/bin/env python3

import struct
import json
import sys

def write_tileset(filename, spec):
    with open(spec['base'], 'rb') as src:
        base_data = src.read()
    with open(spec['overlay'], 'rb') as src:
        overlay_data = src.read()

    if 'cc2' in spec:
        with open(spec['cc2'], 'rb') as src:
            cc2_data = src.read()
    else:
        cc2_data = b''

    tis = open(filename, 'wb')
    tis.write(b'CCTILE02')
    tis.write(struct.pack('I', len(spec['name'])))
    tis.write(bytes(spec['name'], 'utf-8'))
    tis.write(struct.pack('I', len(spec['desc'])))
    tis.write(bytes(spec['desc'], 'utf-8'))
    tis.write(struct.pack('B', spec['size']))
    tis.write(struct.pack('I', len(base_data)))
    tis.write(base_data)
    tis.write(struct.pack('I', len(overlay_data)))
    tis.write(overlay_data)
    tis.write(struct.pack('I', len(cc2_data)))
    tis.write(cc2_data)
    tis.close()

# Generate default tilesets if called from the command line
if __name__ == '__main__':
    if len(sys.argv) < 2:
        print('Usage: {} spec.json [...]'.format(sys.argv[0]))
        sys.exit(1)

    for arg in sys.argv[1:]:
        with open(arg, 'r') as spec_file:
            spec = json.load(spec_file)
        tis_filename = arg.rsplit('.', 1)[0] + '.tis'
        write_tileset(tis_filename, spec)


import sys
import ruamel
from ruamel.yaml import YAML
from datetime import datetime, date, time
from common import *
import fileinput

class NonAliasingRTRepresenter(ruamel.yaml.RoundTripRepresenter):
    def ignore_aliases(self, data):
        return True

def main():
    out = {'races': []}

    yaml = YAML(typ='safe')
    with open('data/common.yaml', 'r') as fi:
        ydat = yaml.load(fi)

    d = date.today()
    year = d.year
    month = d.month + 1
    if month == 13:
        # skip Smarch
        year = year + 1
        month = 1
    day = 1

    try:
        while True:
            d = date(year, month, day)
            for w in ydat['weekly']:
                if w['isoweekday'] == d.isoweekday():
                    t = time.fromisoformat(w['time'])
                    dt = datetime.combine(d, t, tzinfo=RACETZ)
                    r = {'datetime': dt.strftime('%Y-%m-%d %H:%M')}
                    for k in w.keys():
                        if k not in ('isoweekday', 'time', 'msg'):
                            r[k] = w[k]
                    kw = dt.isocalendar().week

                    sunday = d.isoweekday() == 7
                    casual = r['skills_preset'] == 'casual'

                    # alternate simple and complex, and do the opposite on Sunday
                    simple = True
                    if (kw % 2 == 0 and casual) or (kw % 2 != 0 and not casual):
                        simple = False
                    if sunday:
                        simple = not simple

                    if simple:
                        r['desc'] += ' - Simple'
                    else:
                        r['desc'] += ' - Complex'
                    out['races'].append(r)

            day = day + 1

    except ValueError:
        pass

    # ugly hack
    replace_last = 'Casual - Complex' if month % 2 == 0 else 'Hard - Complex'
    for i, obj in reversed(list(enumerate(out['races']))):
        if obj['desc'] == replace_last:
            out['races'][i]['desc'] = obj['desc'].replace('Complex', 'Mystery')
            break

    yout = YAML()
    yout.default_flow_style = False
    yout.version = (1, 2)
    yout.indent(mapping=2, sequence=4, offset=2)
    yout.Representer = NonAliasingRTRepresenter
    with open('data/races-new.yaml', 'w') as fout:
        yout.dump(out, fout)

    # add whitespace
    with fileinput.FileInput('data/races-new.yaml', inplace=True) as f:
        for line in f:
            if line.startswith('  -') and f.lineno() > 4:
                print()
            print(line, end='')

if __name__ == '__main__':
    main()


#!/usr/bin/env python

import argparse
import itertools

import pandas as pd

def getCmdArgs():
    p = argparse.ArgumentParser(description="Add scenes in neighboring rows of scenes in a given scene ID list.")

    p.add_argument("-l", "--list", dest="scn_list", required=True, default=None, metavar="CSV_OF_SCENE_LIST", help="A CSV file of scene list. It must have at least the first column as the list of scene IDs (e.g. LC80010042015211LGN01)")

    p.add_argument("--lead", dest="nlead", required=False, default=1, metavar="NUM_OF_LEADING_ROWS_TO_BUFFER", help="Number of leading rows to buffer from a scene, e.g. 2 leading rows of path=18,row=30 will add two path/row pairs, (1) path=18,row=31, (2) path=18,row=32.")
    p.add_argument("--trail", dest="ntrail", required=False, default=1, metavar="NUM_OF_TRAILING_ROWS_TO_BUFFER", help="Number of trailing rows to buffer from a scene, e.g. 2 leading rows of path=18,row=30 will add two path/row pairs, (1) path=18,row=29, (2) path=18,row=28.")

    p.add_argument("-o", "--output", dest="output", required=True, default=None, metavar="OUTPUT_PRD_LIST", help="Name of output CSV file of the list of path,row,start_date,end_date, of the scenes after row buffering.")

    cmdargs = p.parse_args()

    return cmdargs

def scnIdToPathRowDay(scn_id):
    path = int(scn_id[3:6])
    row = int(scn_id[6:9])
    year = int(scn_id[9:13])
    doy = int(scn_id[13:16])
    return path, row, year, doy

def main(cmdargs):
    scn_csv = cmdargs.scn_list
    nlead = cmdargs.nlead
    ntrail = cmdargs.ntrail
    prd_csv = cmdargs.output

    out_header = ["path", "row", "start_date", "end_date"]
    prd_header = ["path", "row", "year", "doy"]
    with open(prd_csv, "w") as out_fobj:
        out_fobj.write(",".join(out_header))
        out_fobj.write("\n")
        scn_df = pd.read_csv(scn_csv, usecols=[0])
        scn_list = scn_df.iloc[:, 0].tolist()
        prd_list = zip(*[scnIdToPathRowDay(scn) for scn in scn_list])
        prd_dict = {nm:prd for nm, prd in zip(prd_header, prd_list)}
        prd_df = pd.DataFrame(prd_dict)
        # Add buffer rows
        buf_row_add = range(-1*ntrail, nlead+1)
        buf_row_add.remove(0)
        buf_df_list = [prd_df.copy() for bra in buf_row_add]
        for bra, bd in itertools.izip(buf_row_add, buf_df_list):
            bd["row"] = bd["row"] + bra

        all_prd_df = pd.concat([prd_df]+buf_df_list, axis=0)
        all_prd_df = all_prd_df.drop_duplicates(prd_header, keep=False)
        all_prd_df = all_prd_df.sort_values(["year", "doy", "path", "row"])
        datestr = ["{0:04d}{1:03d}".format(getattr(row, "year"), getattr(row, "doy"))
                   for row in all_prd_df.itertuples()]
        all_prd_df[out_header[2]] = pd.to_datetime(datestr, format="%Y%j")
        all_prd_df[out_header[3]] = pd.to_datetime(datestr, format="%Y%j")

        all_prd_df.to_csv(out_fobj, header=False, index=False, columns=out_header,
                          mode="a", date_format="%Y-%m-%d")

if __name__ == "__main__":
    cmdargs = getCmdArgs()
    main(cmdargs)


"""
Device discovery
"""

import enum
import re
from dataclasses import asdict, dataclass
from queue import Empty, Queue
from socket import inet_ntoa
from typing import Dict, Generator, List, Optional

import click
import requests
import usb
from zeroconf import ServiceBrowser, ServiceInfo, ServiceStateChange, Zeroconf

from brewblox_ctl import const, tabular, utils

BREWBLOX_DNS_TYPE = '_brewblox._tcp.local.'
DISCOVER_TIMEOUT_S = 5
DISCOVERY_LEN = 4  # USB / TCP / mDNS
MODEL_LEN = 7  # 'Spark 2' / 'Spark 3' / 'Spark 4'
MAX_ID_LEN = 24  # Spark 4 IDs are shorter
HOST_LEN = 4*3+3

class DiscoveryType(enum.Enum):
    all = 1
    usb = 2
    mdns = 3
    mqtt = 4

    # aliases
    wifi = 3
    lan = 3

    def __str__(self):
        return self.name

    @staticmethod
    def choices():
        return list((str(v) for v in DiscoveryType.__members__))

@dataclass
class DiscoveredDevice:
    discovery: str
    model: str
    device_id: str
    device_host: str = ''

    def __post_init__(self):
        self.device_id = self.device_id.lower()

@dataclass
class HandshakeMessage:
    brewblox: str
    firmware_version: str
    proto_version: str
    firmware_date: str
    proto_date: str
    system_version: str
    platform: str
    reset_reason_hex: str
    reset_data_hex: str
    device_id: str
    model: str = ''

    def __post_init__(self):
        self.device_id = self.device_id.lower()

        if self.platform == 'photon':
            self.model = 'Spark 2'
        elif self.platform == 'p1':
            self.model = 'Spark 3'
        elif self.platform == 'esp32':
            self.model = 'Spark 4'
        else:
            self.model = self.platform

def match_id_services(config: Optional[dict]) -> Dict[str, str]:  # [ID, service_name]
    """Gets the --device-id value for all Spark services in config.

    Because IDs are yielded during discovery,
    values are returned with the ID as key,
    and a comma-separated string of services as value
    """
    if not config:
        return {}

    output: Dict[str, List[str]] = {}
    for name, service in config.get('services', {}).items():
        if not service.get('image', '').startswith('ghcr.io/brewblox/brewblox-devcon-spark'):
            continue
        match = re.match(
            r'.*\-\-device\-id(\w|=)(?P<id>\w+)',
            service.get('command', ''))
        if match:
            id = match.group('id').lower()
            output.setdefault(id, []).append(name)

    return {
        id: ', '.join(services)
        for id, services
        in output.items()
    }

def find_device_by_host(device_host: str) -> Optional[DiscoveredDevice]:
    utils.info(f'Querying device with address {device_host}...')
    try:
        resp = requests.get(f'http://{device_host}', timeout=5)
        resp.raise_for_status()

        content = resp.text
        if not content.startswith('!BREWBLOX'):
            raise RuntimeError('Host did not respond with a Brewblox handshake')

        handshake = HandshakeMessage(*content.split(','))
        utils.info(f'Found a {handshake.model} with ID {handshake.device_id}')
        return DiscoveredDevice(
            discovery='TCP',
            device_id=handshake.device_id,
            model=handshake.model,
            device_host=device_host,
        )

    except Exception as ex:
        utils.warn(f'Failed to fetch device info: {str(ex)}')
        return None

def discover_usb() -> Generator[DiscoveredDevice, None, None]:
    devices = [
        *usb.core.find(find_all=True,
                       idVendor=const.VID_PARTICLE,
                       idProduct=const.PID_PHOTON),
        *usb.core.find(find_all=True,
                       idVendor=const.VID_PARTICLE,
                       idProduct=const.PID_P1),
        # Spark 4 does not support USB control, and is not listed
    ]
    for dev in devices:
        dev: usb.core.Device
        id = usb.util.get_string(dev, dev.iSerialNumber).lower()
        model = {const.PID_PHOTON: 'Spark 2', const.PID_P1: 'Spark 3'}[dev.idProduct]
        yield DiscoveredDevice(discovery='USB',
                               model=model,
                               device_id=id)

def discover_mdns() -> Generator[DiscoveredDevice, None, None]:
    queue: Queue[ServiceInfo] = Queue()
    conf = Zeroconf()

    def on_service_state_change(zeroconf: Zeroconf, service_type, name, state_change):
        if state_change == ServiceStateChange.Added:
            info = zeroconf.get_service_info(service_type, name)
            queue.put(info)

    try:
        ServiceBrowser(conf, BREWBLOX_DNS_TYPE, handlers=[on_service_state_change])
        while True:
            info = queue.get(timeout=DISCOVER_TIMEOUT_S)
            if not info or not info.addresses or info.addresses == [b'\x00\x00\x00\x00']:
                continue  # discard simulators
            id = info.properties[b'ID'].decode()
            model = info.properties[b'HW'].decode()
            host = inet_ntoa(info.addresses[0])
            yield DiscoveredDevice(discovery='mDNS',
                                   model=model,
                                   device_id=id,
                                   device_host=host)
    except Empty:
        pass
    finally:
        conf.close()

def discover_device(discovery_type: DiscoveryType) -> Generator[DiscoveredDevice, None, None]:
    if discovery_type in [DiscoveryType.all,
                          DiscoveryType.usb]:
        yield from discover_usb()
    if discovery_type in [DiscoveryType.all,
                          DiscoveryType.mdns,
                          DiscoveryType.mqtt]:
        yield from discover_mdns()

def list_devices(discovery_type: DiscoveryType,
                 compose_config: Optional[dict]):
    id_services = match_id_services(compose_config)
    table = tabular.Table(
        keys=[
            'discovery',
            'model',
            'device_id',
            'device_host',
            'service'
        ],
        headers={
            'discovery': 'Discovery'.ljust(DISCOVERY_LEN),
            'model': 'Model'.ljust(MODEL_LEN),
            'device_id': 'Device ID'.ljust(MAX_ID_LEN),
            'device_host': 'Device host'.ljust(HOST_LEN),
            'service': 'Service',
        }
    )

    utils.info('Discovering devices...')
    table.print_headers()
    for dev in discover_device(discovery_type):
        table.print_row({
            **asdict(dev),
            'service': id_services.get(dev.device_id, ''),
        })

def choose_device(discovery_type: DiscoveryType,
                  compose_config: Optional[dict],
                  ) -> Optional[DiscoveredDevice]:
    id_services = match_id_services(compose_config)
    table = tabular.Table(
        keys=[
            'index',
            'discovery',
            'model',
            'device_id',
            'device_host',
            'service'
        ],
        headers={
            'index': 'Index',
            'discovery': 'Discovery'.ljust(DISCOVERY_LEN),
            'model': 'Model'.ljust(MODEL_LEN),
            'device_id': 'Device ID'.ljust(MAX_ID_LEN),
            'device_host': 'Device host'.ljust(HOST_LEN),
            'service': 'Service',
        }
    )
    devs = []

    utils.info('Discovering devices...')
    table.print_headers()
    for dev in discover_device(discovery_type):
        # TODO(Bob) less hacky check
        if discovery_type == DiscoveryType.mqtt and dev.model != 'Spark 4':
            continue
        devs.append(dev)
        table.print_row({
            **asdict(dev),
            'index': len(devs),
            'service': id_services.get(dev.device_id, ''),
        })

    if not devs:
        click.echo('No devices discovered')
        return None

    idx = click.prompt('Which device do you want to use?',
                       type=click.IntRange(1, len(devs)),
                       default=1)

    return devs[idx-1]


import itertools
import os
import pandas as pd
import pickle
import random
import torch
from torchvision import datasets
from tqdm import tqdm
from scipy import rand

import data_handler
from models import AlexNet, Vgg16

def generate_model_testset_results(model, testset_path):
    """
    Evaluate whole 'imagenetv2-matched-frequency-format-val' dataset
    on given model and saves the results ("img_name", "max_confidence", "pred_label", "true_label" for each image)
    in a DataFrame via pickle.

    For information on the dataset see https://github.com/modestyachts/ImageNetV2
    """
    img_folder = datasets.ImageFolder(root=testset_path)

    img_names, true_labels_idx, pred_labels_idx, pred_max_confidences = [], [], [], []

    for img_path in tqdm(img_folder.imgs):
        pil_img = img_folder.loader(img_path[0])
        img_name = img_path[0].split(os.sep)[-1]

        # preprocessing and prediction
        input_tensor = data_handler.transform()(pil_img)
        input_tensor = input_tensor.unsqueeze(0)
        output = model.predict(input_tensor)
        # output has unnormalized scores. To get probabilities, run a softmax on it.
        probabilities = torch.nn.functional.softmax(output[0], dim=0)

        img_names.append(img_name)
        pred_max_confidences.append(probabilities.detach().numpy().max())
        pred_labels_idx.append(probabilities.detach().numpy().argmax())
        true_labels_idx.append(int(img_path[0].split(os.sep)[-2]))

    df = pd.DataFrame([img_names, pred_max_confidences, pred_labels_idx, true_labels_idx]).transpose()
    df.columns = ["img_name", "max_confidence", "pred_label", "true_label"]
    df["pred_is_correct"] = df["pred_label"] == df["true_label"]

    return df

def create_questionnairs(imgs_idx, xai_methods, model_names, df_vgg, df_alex, seed=None):
    if seed:
        random.seed(seed)
    # create first half of question with fixed images for all questionnaire forms
    questionnaires_list = get_fixed_img_questionnaires(imgs_idx, xai_methods, model_names)
    # adding images works directly on the reference of 'questionnaires_list'
    add_random_unique_images(questionnaires_list, imgs_idx, df_alex, df_vgg, model_names, xai_methods)

    return questionnaires_list

def get_fixed_img_questionnaires(imgs_idx, xai_methods, models):
    NUM_QUESTIONNAIRES = 12
    NUM_IMGS = 12
    questionnaires_list = []
    random_imgs_idx = [imgs_idx.pop(random.randint(0, len(imgs_idx) - 1)) for i in range(NUM_IMGS)]
    permutations = list(itertools.product(random_imgs_idx, models, xai_methods))
    # distribute permutations on questionnaires
    for q in range(NUM_QUESTIONNAIRES):
        questionnaire = []
        for i in range(NUM_IMGS):
            if (q + i) > (NUM_IMGS - 1):
                questionnaire.append(permutations[i * NUM_IMGS:i * NUM_IMGS + NUM_IMGS][(q + i) - NUM_IMGS])
            else:
                questionnaire.append(permutations[i * NUM_IMGS:i * NUM_IMGS + NUM_IMGS][q + i])
        questionnaires_list.append(questionnaire)

    return questionnaires_list

def add_random_unique_images(questionnaires_list, imgs_idx, df_alex, df_vgg, model_names, xai_methods):
    FINAL_QUESTIONNAIRE_SIZE = 24

    for idx_qn, questionnaire in enumerate(questionnaires_list):

        df_variants_count = pd.DataFrame(list(itertools.product(xai_methods, model_names, [True, False]))).groupby(
            [0, 1, 2]).count()
        df_variants_count["count"] = 0

        # evaluate variants for the already drawn fixed questions
        for idx_q, question in enumerate(questionnaire):
            if question[1] == "alex":
                if df_alex["pred_is_correct"][question[0]]:
                    questionnaires_list[idx_qn][idx_q] += (True,)
                    df_variants_count.loc[question[2], "alex", True]["count"] += 1
                else:
                    questionnaires_list[idx_qn][idx_q] += (False,)
                    df_variants_count.loc[question[2], "alex", False]["count"] += 1
            else:
                if df_vgg["pred_is_correct"][question[0]]:
                    questionnaires_list[idx_qn][idx_q] += (True,)
                    df_variants_count.loc[question[2], "vgg", True]["count"] += 1
                else:
                    questionnaires_list[idx_qn][idx_q] += (False,)
                    df_variants_count.loc[question[2], "vgg", False]["count"] += 1
        """
        add addtional random images to each questionnaire such that for every variant in df_variants_count the
        count will be 1
        """
        while df_variants_count["count"].sum() != FINAL_QUESTIONNAIRE_SIZE:
            rand_img_idx = imgs_idx.pop(random.randint(0, len(imgs_idx) - 1))

            alex_pred = df_alex.loc[rand_img_idx]["pred_is_correct"]
            vgg_pred = df_alex.loc[rand_img_idx]["pred_is_correct"]

            df_alex_options = df_variants_count.loc[:, "alex", alex_pred]
            df_alex_options = df_alex_options[df_alex_options["count"] == 0]

            df_vgg_options = df_variants_count.loc[:, "vgg", vgg_pred]
            df_vgg_options = df_vgg_options[df_vgg_options["count"] == 0]

            if not df_alex_options.empty:
                rand_variant = df_alex_options.index[random.randint(0, df_alex_options.shape[0] - 1)]
                question = (rand_img_idx, rand_variant[1], rand_variant[0], rand_variant[2])
                questionnaire.append(question)
                df_variants_count.loc[rand_variant]["count"] += 1

            elif not df_vgg_options.empty:
                rand_variant = df_vgg_options.index[random.randint(0, df_vgg_options.shape[0] - 1)]
                question = (rand_img_idx, rand_variant[1], rand_variant[0], rand_variant[2])
                questionnaire.append(question)
                df_variants_count.loc[rand_variant]["count"] += 1

def save_questionnaires(questionnaires_list, path):
    with open(path, 'wb') as f:
        pickle.dump(questionnaires_list, f)

def shuffle_questions(questionnaire):
    for questionnaire in questionnaire:
        random.shuffle(questionnaire)

def main():
    """
    create questionnaires
    must only be evaluated if testset hasn't already been evaluated
    """
    folder_vgg = os.path.join(os.path.curdir, "data", "stats", "df_vgg.pickle")
    folder_alex = os.path.join(os.path.curdir, "data", "stats", "df_alexnet.pickle")
    if not (os.path.exists(folder_alex) and os.path.exists(folder_vgg)):
        models = [Vgg16(), AlexNet()]

        for model in models:
            model.eval()
            folder = os.path.join(os.path.curdir, "data", "imagenetv2-matched-frequency-format-val")
            df = generate_model_testset_results(model, folder)
            df.to_pickle(f"data/stats/df_{model.name}_2.pickle")

        imgs_idx = list(range(10000))
        xai_methods = ['gradCAM', 'LRP', 'SHAP', 'LIME', 'ConfidenceScores', 'IntegratedGradients']
        model_names = ["alex", "vgg"]

        df_vgg = pd.read_pickle(folder_vgg)
        df_alex = pd.read_pickle(folder_alex)

        questionnaires_list = create_questionnairs(imgs_idx, xai_methods, model_names, df_vgg, df_alex, seed=3)
        shuffle_questions(questionnaires_list)
        folder = os.path.join(os.path.curdir, "data", "question_generation", "questionnaires_shuffled.pickle")
        save_questionnaires(questionnaires_list, folder)

    # additionally shuffle questions in questionnairs
    folder = os.path.join(os.path.curdir, "data", "question_generation", "questionnaires.pickle")
    questionnaires_list = data_handler.get_questionnaires(folder)

    shuffle_questions(questionnaires_list)

    folder = os.path.join(os.path.curdir, "data", "question_generation", "questionnaires_shuffled.pickle")
    save_questionnaires(questionnaires_list, folder)

if __name__ == '__main__':
    main()


"""empty message

Revision ID: c4665b8d682b
Revises: 10dbb0e0a903
Create Date: 2019-12-26 14:38:11.609539

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'c4665b8d682b'
down_revision = '10dbb0e0a903'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('goods', sa.Column('gage', sa.Integer(), nullable=True))
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('goods', 'gage')
    # ### end Alembic commands ###


"""Unit tests for the config module."""

import os
import pytest
from wmtmetadata.config import Config, ConfigFromFile, ConfigFromHost
from wmtmetadata.host import HostInfo
from . import data_dir

tmp_dir = '/tmp'
sample_config_file = os.path.join(data_dir, 'wmt-config-siwenna.yaml')
host = 'siwenna.colorado.edu'
name = 'Hydrotrend'
fetched_config_file = 'wmt-config-{}.yaml'.format(host)

def test_config():
    config = Config()
    assert isinstance(config, Config)

def test_configfromfile():
    config = ConfigFromFile(sample_config_file)
    assert config.filename == sample_config_file

def test_configfromfile_load():
    config = ConfigFromFile(sample_config_file)
    config.load()
    components = config.components.keys()
    assert components.pop() == name
    assert config.host['hostname'] == host

def test_configfromhost():
    config = ConfigFromHost(host)
    assert config.executor.info['name'] == host

@pytest.mark.skip(reason="Don't abuse remote test machine")
def test_configfromhost_build():
    config = ConfigFromHost(host)
    config.build()

@pytest.mark.skip(reason="Don't abuse remote test machine")
def test_configfromhost_fetch():
    config = ConfigFromHost(host)
    config.fetch(local_dir=tmp_dir)
    assert os.path.isfile(os.path.join(tmp_dir, fetched_config_file))

@pytest.mark.skip(reason="Don't abuse remote test machine")
def test_configfromhost_load():
    config = ConfigFromHost(host)
    config.build()
    config.fetch(local_dir=tmp_dir)
    config.load()
    components = config.components.keys()
    assert components.pop() == name


'''
函数说明:
Author: hongqing
Date: 2021-08-04 14:23:54
LastEditTime: 2021-08-04 15:23:25
'''
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
numoffinger=21
class Net(nn.Module):
    def __init__(self,type=1):
        super(Net, self).__init__()
        self.fc1 = nn.Linear(numoffinger-1, 255)
        self.fc2 = nn.Linear(255, 255)
        self.fc3 = nn.Linear(255, 3)
        if(type==1):
            self.fc1.weight.data.normal_(0, 3)   # initialization
            self.fc2.weight.data.normal_(0, 3)
            self.fc3.weight.data.normal_(0, 3)
        if(type==0):
            self.fc1.weight.data.zero_()   # initialization
            self.fc2.weight.data.zero_()
            self.fc3.weight.data.zero_()
        if(type==2):
            self.fc1.weight.data.random_(1,2)   # initialization
            self.fc2.weight.data.random_(1,2)
            self.fc3.weight.data.random_(1,2)
    def forward(self, x):
        x = self.fc1(x)
        x = F.relu(x)
        x = self.fc2(x)
        x = F.relu(x)
        x = self.fc3(x)
        return x


import glob
import os
import random
import shutil

import numpy as np
import torch
import torch.nn as nn
import torchvision.transforms.v2 as T

def set_seed(seed: int = 42):
    """Sets the seed for reproducibility."""
    random.seed(seed)
    np.random.seed(seed)
    os.environ["PYTHONHASHSEED"] = str(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = False

def remove_glob(pathname: str):
    for p in glob.glob(pathname, recursive=True):
        if os.path.isfile(p):
            os.remove(p)
        elif os.path.isdir(p):
            shutil.rmtree(p)

class ImageTransform:
    """Image transformation module."""

    def __init__(self, input_size=384, phase="train"):
        if phase == "train":
            self.data_transform = nn.Sequential(
                T.RandomResizedCrop(input_size, (0.25, 1.0), (3 / 4, 4 / 3)),
                T.RandomChoice(
                    [
                        T.RandomRotation((0, 0)),
                        T.RandomRotation((90, 90)),
                        T.RandomRotation((180, 180)),
                        T.RandomRotation((270, 270)),
                    ],
                ),
                T.RandomHorizontalFlip(p=0.5),
                T.ColorJitter(brightness=0.5),
                T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
            )
        elif phase == "test":
            self.data_transform = nn.Sequential(
                T.Resize(input_size),
                T.CenterCrop(input_size),
                T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
            )

    def __call__(self, img):
        return self.data_transform(img)


import requests
import json
import urllib3
from settings import settings as settings
from sdwan_operations import monitor as sdwanmn
import time
import sys, getopt

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

def main(argv):
    # To run the program use the syntax:
    # python __main__.py -d <devfile> -f <fieldfile> -c <optional csvfile>
    devfile = ''
    fieldfile = ''
    csvfile = ''
    try:
        opts, args = getopt.getopt(argv,"hd:f:c:",["devfile=", "fieldfile=", "csvfile"])
    except getopt.GetoptError:
        print ('python __main__.py -d <devfile> -f <fieldfile> -c <optional csvfile>')
        sys.exit(2)
    for opt, arg in opts:
        if opt == '-h':
            print('python __main__.py -d <devfile> -f <fieldfile> -c <optional csvfile>')
            sys.exit(1)
        elif opt in ("-d", "--devfile"):
            devfile = arg
        elif opt in ("-f", "--fieldfile"):
            fieldfile = arg
        elif opt in ("-c", "--csvfile"):
            csvfile = arg

    # Read the device list from the supplied file (for example, dev_list.json)
    try:
        with open(devfile) as f:
            dev_list = json.load(f)
    except FileNotFoundError:
        print ('python __main__.py -d <devfile> -f <fieldfile> -c <optional csvfile>')
        sys.exit(2)

    # Read the field list from the supplied file (for example, field_list.json)
    try:
        with open(fieldfile) as f:
            field_list = json.load(f)
    except FileNotFoundError:
        print ('python __main__.py -d <devfile> -f <fieldfile> -c <optional csvfile>')
        sys.exit(2)
    while (True):
        # Create a session and pass it as a parameter for show_port_stats
        session = requests.Session()
        session.auth = (settings.vmanage_username, settings.vmanage_password)
        sdwanmn.show_port_stats(session, dev_list, field_list)
        # Comment below to stop exporting data to csv
        if csvfile != "":
            sdwanmn.dump_to_csv(session,dev_list,field_list,csvfile)
        time.sleep(60)

if __name__ == "__main__":
    main(sys.argv[1:])


import logging
import os
from copy import deepcopy

from harmony.util import shortened_id
from harmony.repository_state import RepositoryState

logger = logging.getLogger(__name__)

def commit(local_location_id, working_directory, location_states, repository_state):
    """
    Scan the given working directory for changes and commit them to local
    state storage.
    That is, update location_states[local_location_id] with the
    new current file states (digests, "who has what?").
    Also update repository_state info ("who made which content decision in what
    order?")

    Parameters:

    local_location_id:
        ID of the location that is considered local (i.e. the one that belongs
        to the working_directory instance)

    working_directory:
        WorkingDirectory instance representing the local working directory.

    location_states:
        LocationStates instance representing the local location state storage.
        Will (possibly) be modified.

    repository_state:
        RepositoryState instance representing the local repository state
        storage. Will (possibly) be modified.

    return:
        True iff any change was recorded.
    """

    id_ = local_location_id
    short_id = shortened_id(id_)

    paths = set(working_directory.get_filenames()) \
            | set(location_states.get_all_paths(id_))

    # 1. update location state
    #    - detect renames (add WIPE entries later for those)
    #    - when a file is *added* that is known to other locations w/
    #      different digest, let user confirm what he wants to do (see
    #      above)
    #    - increase local clock
    #
    # 2. update repository state
    #    - if file changed in step 1:
    #      clock = current clock for local + max for each other location
    #      hash = current local hash
    #      (deviate from this if user selected to do something else)
    #    - if file did not change:
    #      no change in hash or clock

    # Do all the file scanning before so we can be sure to do it at most
    # once per file in the WD
    wd_states = {
        path: working_directory.generate_file_state(path)
        for path in paths
        if working_directory.file_maybe_modified(
            location_states.get_file_state(id_, path)
        )
    }

    location_state_cache = {
        path: location_states.get_file_state(id_, path)
        for path in paths
    }

    any_change = False
    for path in paths:

        if path in wd_states:
            file_state = location_state_cache[path]
            new_file_state = wd_states[path]
            changed = location_states.update_file_state(id_, new_file_state)
            if changed:
                any_change = True

                # If the file vanished but a new one with the same digest
                # popped up, consider that a rename.
                # Rename means, the old file is WIPEd (instead of just
                # locally removed) and the new file is added as usual
                if not new_file_state.exists():
                    logger.debug('{} vanished'.format(new_file_state.path))

                    # Iterate over paths to find a possible rename target
                    for path2 in paths:
                        # Rename to itself does not make sense
                        # Rename to a file that has not changed (or better: just appeared) does not make sense
                        if path2 == path or path2 not in wd_states:
                            continue

                        path2_state = location_state_cache[path2]
                        new_path2_state = wd_states[path2]
                        logger.debug('{} rename candidate {} ex before={} ex now={} self.digest={} candidate.digest={}'.format(
                            path, path2, path2_state.exists(),
                            new_path2_state.exists(),
                            file_state.digest, new_path2_state.digest
                        ))

                        if not path2_state.exists() \
                           and new_path2_state.exists() \
                           and new_path2_state.digest == file_state.digest:
                            logger.info('Detected rename: {} -> {}'.format(path, path2))
                            new_file_state.wipe = True
                            new_file_state.digest = file_state.digest
                            break

                repository_state.update_file_state(
                    new_file_state,
                    id_,
                    location_states.get_clock(id_) + 1,
                )
                logger.debug('{} committed: {} clk={}'.format(short_id, new_file_state.path, location_states.get_clock(id_) + 1))
            else:
                logger.debug('{} not actually changed: {}'.format(short_id, path))
        else:
            logger.debug('{} not changed: {}'.format(short_id, path))

    return any_change

def merge(local_state, remote_state, merger_id):
    """
    Merge two repository states ('local' and 'remote') into a common state if
    possible, auto-detecting if a change only happened on one side and
    propagating those changes.
    For cases in which a file was changed on both sides, return details of the
    conflict.

    local_state:
        RepositoryState() instance that reflects the local repository state.
    remote_state:
        RepositoryState() instance that reflects the remote repository state.
    merger_id:
        ID of the repository conducting the merge (assumed to correspond
        to the 'local' repository)
    return:
        A pair (conflicts, merged).
        $conflicts is a dictonary of the form { path: (local_entry, remote_entry),
        ... } whereas $path denotes the path of a file in conflict and $local_entry
        and $remote_entry refer to the RepositoryState.Entry instances for that
        file that are in conflict.
        $merged is a newly created RepositoryState instance with selected merged
        repository states.
        If $conflicts is empty, $merged covers all files present either locally or
        remotely.
    """
    local_paths = set(local_state.get_paths())
    remote_paths = set(remote_state.get_paths())

    merged = RepositoryState(None)
    conflicts = {}

    for p in local_paths - remote_paths:
        merged[p] = local_state[p]

    for p in remote_paths - local_paths:
        merged[p] = remote_state[p]

    # conflicts can only arise in paths that are specified in both state
    # files
    paths = set(local_state.get_paths()) & set(remote_state.get_paths())

    for path in paths:
        local = local_state[path]
        remote = remote_state[path]

        c = local.clock.compare(remote.clock)
        if c is None:
            if local.contents_different(remote):
                logger.debug('merge: {} in conflict: {} <-> {}'.format(
                    path, local.clock, remote.clock
                ))
                conflicts[path] = (local, remote)
            else:
                logger.debug('merge: {} automerged (same content)'.format(path))
                m = deepcopy(local)
                m.clock.update(remote.clock)
                m.clock.increase(merger_id)
                merged[path] = m

        elif c < 0:
            logger.debug('merge: {} newer on remote'.format(path))
            merged[path] = remote

        else: # c >= 0:
            logger.debug('merge: {} same version or newer on local'.format(path))
            merged[path] = local

    return conflicts, merged

def auto_rename(working_directory, repository_state):
    from harmony.working_directory import WorkingDirectory

    assert isinstance(working_directory, WorkingDirectory)
    """
    Apply automatic renaming in the given working_directory.
    That is, if working dir contains files that are WIPEd in $repository_state but
    are present under a different name, automatically rename those to obtain
    the repository file at a low cost.

    Repository.commit() should be called after calling this to commit the
    changes to the working directory.

    precondition: WD clean
    """

    # Automatically apply auto-renaming
    # Auto-renaming
    # -------------
    # 1. Find any files $A with a WIPE entry.
    # 2. Compute/get their digest (from location state)
    # 3. Find a non-wiped file $B in repo that does not exist in the WD
    # 4. Rename $A to $B

    for path, entry in repository_state.files.items():
        logger.debug('auto_rename: {}: path={} wipe={} in_wd={}'.format(path, entry.path, entry.wipe, (entry.path in working_directory)))
        if entry.wipe and (entry.path in working_directory):
            possible_targets = {
                e.path for e in repository_state.files.values()
                if e.path != path and e.digest == entry.digest and not e.wipe
            }
            logger.info(
                '{} could be auto-renamed to any of {}'.format(
                    path, possible_targets
                )
            )
            if possible_targets:
                (working_directory.path / path).rename(working_directory.path / possible_targets.pop())


from django.db import models

# Create your models here.

class Meal(models.Model):
    menu = models.TextField(blank=True)

    def __str__(self):
        return self.menu

class MealList(models.Model):
    date = models.DateField(blank=True)

    breakfast = models.ForeignKey(
        Meal,
        blank=True,
        null=True,
        on_delete=models.CASCADE,
        related_name="breakfast_id",
    )
    lunch = models.ForeignKey(
        Meal,
        null=True,
        blank=True,
        on_delete=models.CASCADE,
        related_name="lunch"
    )
    dinner = models.ForeignKey(
        Meal,
        null=True,
        blank=True,
        on_delete=models.CASCADE,
        related_name="dinner"
    )

    created_at = models.DateTimeField(auto_now_add = True)
    updated_at = models.DateTimeField(auto_now = True)

    class Meta:
        db_table = 'meal_lists'

    def update_meal(self, data, number):
        if number == 1:
            self.breakfast = data
        if number == 2:
            self.lunch = data
        if number == 3:
            self.dinner = data
        self.save()
        return self


from os import path
from subprocess import check_output, check_call
from pathlib import Path

def _filename(fname):
    import yapl
    yapl_root = Path(yapl.__file__).parent
    filename = fname + ".sh"
    full_filename = yapl_root / filename
    return str(full_filename)

def func(fname, *args):
    shell_script = _filename(fname)
    command = list([shell_script] + [str(a) for a in args])
    result = check_output(command, text=True).strip()
    return result

def proc(fname, *args):
    shell_script = _filename(fname)
    command = list([shell_script] + [str(a) for a in args])
    check_call(command)


from scipy import linalg
import numpy as np
import scipy.optimize as sopt
import matplotlib.pyplot as plt
from scipy.optimize import fsolve
from scipy.optimize import LinearConstraint
from matplotlib import cm
import tabulate
import math

START_POINT = [2,2]

# fun - main muction
def fun(x: np.ndarray) -> np.float64:
    return ((x[0]-2)**2) + (x[1]-1)**2

# hfunc - represents an "= 0" fun
def hfun(x: np.ndarray) -> np.float64:
    return x[0]-2*x[1]+1

# gfunc - represents an ">= 0" fun
def gfun(x: np.ndarray) -> np.float64:
    return -0.25 * x[0]**2 - x[1]**2+1

def gplusfun(g, x, u, c):
    return min(g(x), np.divide(u, c))

def modified_lagrange_method(fun, points, epsx, g_constarins, h_constrains, u=[0], a=[0], c=0.1, beta=2, counter=0, func_counter = 0, _callback=None):
    """Minimize a function with given constrains
    Arguments:
        points {[float]} -- [array of calculated points]
        epsx {[float]} -- [epsilon]
        g_constarins {[Callable]} -- [array of inequality constrains]
        h_constrains {[Callable]} -- [array of equality constrains]
    Keyword Arguments:
        u {list} -- [Langrange factor for inequality, must be same length as g_constarins] (default: {[0]})
        a {list} -- [Langrange factor for equality, must be same length as h_constrains] (default: {[0]})
        c {float} -- [penalty factor] (default: {0.1})
        beta {int} -- [growth rate of penalty factor must be in range [2;26]] (default: {2})
        counter {int} -- [counter] (default: {0})
        callback - function that takes dict x, witch contains all intermediate values such as x, u, a, c, f(x), L(x,u,a,c)
    """
    def lagrange(x):
        if(len(g_constarins) != 0 and len(g_constarins) != 0):
            array_of_constrains_g = np.array(
                [gplusfun(g_constrain, x, u_i, c) for g_constrain, u_i in zip(g_constarins, u)])
            array_of_constrains_h = np.array(
                [h_constrain(x) for h_constrain in h_constrains])
            return fun(x) - sum([u_i * g for u_i, g in zip(u, array_of_constrains_g)]) + 0.5*sum(c * array_of_constrains_g**2) - sum([a_i * g for a_i, g in zip(a, array_of_constrains_h)]) + 0.5*sum(c * array_of_constrains_h**2)
        elif(len(h_constrains) != 0 and len(g_constarins) == 0):
            array_of_constrains_h = np.array(
                [h_constrain(x) for h_constrain in h_constrains])
            return fun(x) - sum([a_i * h for a_i, h in zip(a, array_of_constrains_h)]) + 0.5*sum(c * array_of_constrains_h**2)
        elif(len(h_constrains) == 0 and len(g_constarins) != 0):
            array_of_constrains_g = np.array(
                [gplusfun(g_constrain, x, u_i, c) for g_constrain, u_i in zip(g_constarins, u)])
            return fun(x) - sum([u_i * g for u_i, g in zip(u, array_of_constrains_g)]) + 0.5*sum(c * array_of_constrains_g**2)
        else:
            return fun(x)

    if _callback is not None:
        _callback({"x": points[-1], "u": u, "a": a, "c": c, "f": fun(points[-1]), "L": lagrange(points[-1]), "iter": counter, "fiter": func_counter})

    # BFGS - is most fast & eficient for my cases
    res = sopt.minimize(
        lagrange, x0=points[-1], method='BFGS')

    next_val = res.x
    func_counter = func_counter+res.nfev
    counter = counter+res.nit
    points.append(next_val)

    u = np.array([max(0, (u_i - c*g_constrain(next_val)))
                  for g_constrain, u_i in zip(g_constarins, u)])

    a = np.array([a_i-c*h_constrain(next_val)
                  for h_constrain, a_i in zip(h_constrains, a)])

    c = beta*c

    counter = counter+1

    if(abs(next_val - points[-2])[0] < epsx and abs(next_val - points[-2])[1] < epsx):
        return points
    else:
        return modified_lagrange_method(fun, points, epsx, g_constarins, h_constrains, u, a, c, beta, counter, func_counter,_callback)

def filter_zLim(X,Y,Z, zlim):
    for i in range(0, len(Z)):
        for j in range(0, len(Z)):
            if Z[i][j] > zlim[1] or Z[i][j] < zlim[0]:
                Z[i][j] = 4

    return X, Y, Z

def printDecorator(f, res):
    def wrapper(x):
        res.append(x)
        ret_val = f(x)
        return ret_val
    return wrapper

def trunc(number, digits) -> float:
    stepper = 10.0 ** digits
    return math.trunc(stepper * number) / stepper

def plotting():
    xs = []
    results_list = []
    f = lambda x: xs.append(x["x"])
    callback = printDecorator(f, results_list)

    #Adjust plotting scale here where x in [a1,b1] and y in [a2,b2] [a1: b1: 20j, a2: b2: 20j]
    X, Y = np.mgrid[2.4:0:20j, 2.5:-1.5:20j]
    Z = fun(np.array([X, Y]))

    fig = plt.figure()

    ax = fig.add_subplot(1, 2, 1, projection='3d')
    ax.set_xlim((0, 3))
    ax.set_ylim((-1.5, 3))
    ax.set_zlim((0, 4))

    ax.plot_wireframe(X, Y, Z)

    ax.contour(X, Y, gfun(np.array([X, Y])), levels=[0], colors='blue')
    ax.contour(X, Y, hfun(np.array([X, Y])), levels=[0], colors='lime')

    #Put list of constrains here, for my case its one constrain g(x) and one h(x)
    g_constarins = np.array([gfun])
    h_constrains = np.array([hfun])

    vals = modified_lagrange_method(fun, list([START_POINT, ]), 1e-6,
                                    g_constarins, h_constrains, _callback=callback)

    #Print Results Table
    header = results_list[0].keys()
    rows =  [x.values() for x in results_list[0:11] + [results_list[-1]]]
    print(tabulate.tabulate(rows, header, tablefmt='grid'))

    ax.plot(np.array(vals).T[0], np.array(vals).T[1], np.array(
        list(map(fun, np.array(vals)))).T, "x-", color='red')

    ax1 = fig.add_subplot(1, 2, 2)

    #Adjust plotting scale here where x in [a1,b1] and y in [a2,b2] [a1: b1: 20j, a2: b2: 20j]
    X, Y = np.mgrid[3: 0: 20j, 2.3: -1.5: 20j]
    Z = fun(np.array([X, Y]))

    ax1.contour(X, Y, Z, levels=40)

    t = 0
    for x in zip(np.array(vals).T[0], np.array(vals).T[1]):
        if abs(fun(x) - t) > 1e-2:
            ax1.annotate(trunc(fun(x),3), (x[0], x[1]))
        t = fun(x)

    ax1.plot(np.array(vals).T[0], np.array(vals).T[1], "x-", color='red')

    for idx, g_constr in enumerate(g_constarins):
        ax1.clabel(ax1.contour(X, Y, g_constr(np.array([X, Y])), levels=[0], colors='blue'), fmt=f"g{idx}(x)", fontsize=10)

    for idx, h_constr in enumerate(h_constrains):
        ax1.clabel(ax1.contour(X, Y, h_constr(np.array([X, Y])), levels=[0], colors='lime'), fmt=f"h{idx}(x)", fontsize=10)

    plt.show()

plotting()


import os, sys

from langchain.llms import OpenAI
from langchain.chains.question_answering import load_qa_chain
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.vectorstores import FAISS

prefix, query = sys.argv[1:3]

api_key = os.environ.get('OPENAI_API_KEY')
if api_key is None:
    sys.exit("OPENAI_API_KEY is unset")

embeddings = OpenAIEmbeddings(openai_api_key=api_key)
llm = OpenAI(temperature=0.7, openai_api_key=api_key)
qa_chain = load_qa_chain(llm, chain_type="stuff")

fdir = os.path.dirname(prefix)
fname = os.path.basename(prefix)
faiss_index = FAISS.load_local(fdir, embeddings, fname)

# This model's maximum context length is 4097 tokens, 256 for the completion
ss = faiss_index.similarity_search(query.strip(), k=5)
ans = qa_chain.run(input_documents=ss, question=query)

print(ans.strip())


import os, random
import numpy as np
import torch
import argparse

from train import train

def init_seeds(seed=0):
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)

def parsing_args(c):
    parser = argparse.ArgumentParser(description='msflow')
    parser.add_argument('--mode', default='train', type=str,
                        help='train or test.')
    parser.add_argument('--resume', action='store_true', default=False,
                        help='resume training or not.')
    parser.add_argument('--eval_ckpt', default='', type=str,
                        help='checkpoint path for evaluation.')
    parser.add_argument('--gpu', default='0', type=str)
    parser.add_argument('--class-name', default='bottle', type=str)
    parser.add_argument('--lr', default=1e-4, type=float,
                        help='learning rate')
    parser.add_argument('--batch-size', default=8, type=int,
                        help='train batch size')
    parser.add_argument('--meta-epochs', default=25, type=int,
                        help='number of meta epochs to train')
    parser.add_argument('--sub-epochs', default=4, type=int,
                        help='number of sub epochs to train')
    parser.add_argument('--extractor', default='wide_resnet50_2', type=str,
                        help='feature extractor')
    parser.add_argument('--pool-type', default='avg', type=str,
                        help='pool type for extracted feature maps')
    parser.add_argument('--parallel-blocks', default=[2, 5, 8], type=int, metavar='L', nargs='+',
                        help='number of flow blocks used in parallel flows.')
    parser.add_argument('--pro-eval', action='store_true', default=False,
                        help='evaluate the pro score or not.')
    parser.add_argument('--pro-eval-interval', default=4, type=int,
                        help='interval for pro evaluation.')

    args = parser.parse_args()

    for k, v in vars(args).items():
        setattr(c, k, v)

    c.input_size = (256, 256) if c.class_name == 'transistor' else (512, 512)

    return c

def main(c):
    c = parsing_args(c)
    os.environ['CUDA_VISIBLE_DEVICES'] = c.gpu
    init_seeds(seed=c.seed)
    c.version_name = 'msflow_{}_{}pool_pl{}'.format(c.extractor, c.pool_type, "".join([str(x) for x in c.parallel_blocks]))
    c.ckpt_dir = os.path.join(c.work_dir, c.version_name, c.class_name)
    train(c)

if __name__ == '__main__':
    import default as c
    main(c)


#!/usr/bin/python
######################################################################################
## Find disk usage under a root directory, examples:
##   1. python du_rootdir.py -d rootDir
##      Find disk usage under the root directory "rootDir"
##   2. python du_rootdir.py -d rootDir -r true
##      Find disk usage recursively under the root directory "rootDir"
##   3. python du_rootdir.py -d rootDir -r true -l 3
##      Find disk usage recursively under the root directory "rootDir" with
##      recursive level 3
## Author: Zhichang Guo, email: Zhichang.Guo@noaa.gov
######################################################################################
from pathlib import Path
import argparse
import os
import sys

def print_indented(result, level):
    print('\t' * level + result)

def traverse_dir(dir):
    iflag = 'not'
    l = os.listdir(dir)
    for d in l:
        if os.path.isdir(dir + d):
            if not d.startswith('.') or (d.startswith('.') and not iflag.upper() == 'NOT'):
                fullpath = os.path.join(dir, d)
                cmd = "du -sh " + " \'" + str(fullpath) + "\'"
#               print(cmd)
                result = os.popen(cmd).read()
#               print(result)
                print(result,end='')

def traverse_dir_recur(dir, max_level, level=0):
    iflag = 'not'
    l = os.listdir(dir)
    for d in l:
        if os.path.isdir(dir + d):
            if not d.startswith('.') or (d.startswith('.') and not iflag.upper() == 'NOT'):
                fullpath = os.path.join(dir, d)
                if max_level == 0 or level < max_level:
                    traverse_dir_recur(dir + d + "/", max_level, level+1)
                    cmd = "du -sh " + " \'" + str(fullpath) + "\'"
#                   print(cmd)
                    result = os.popen(cmd).read()
#                   print_indented(result.rstrip("\n"), level+1)
                    if level > 0:
                        print('\033[1;31;43m ' + result.rstrip("\n") + ' \033[0;0m')
                    else:
                        print(result.rstrip("\n"))
                    #print(result)

def find_du(rootdir, rflag, max_level):
    home = os.environ['HOME']
    path = Path(rootdir)
    owner = path.owner()
    print("Owner: ",owner)
    rootdir += '/'
    if rflag == 'false':
        traverse_dir(rootdir)
    else:
        traverse_dir_recur(rootdir, max_level)

if __name__ == "__main__":
    ap = argparse.ArgumentParser()
    ap.add_argument('-d', '--rootd', help="name of the root directory", required=True)
    ap.add_argument('-r', '--rflag', help="recurively or not", default="false")
    ap.add_argument('-l', '--level', help="level", type=int, default=0)
    MyArgs = ap.parse_args()
    find_du(MyArgs.rootd, MyArgs.rflag, MyArgs.level)


from pathlib import Path
import pandas as pd
from sklearn.metrics import make_scorer, accuracy_score, f1_score, roc_auc_score
from imblearn.metrics import geometric_mean_score, sensitivity_score, specificity_score

def get_slovak_data(business_area, year, postfix):
    print("Loading Slovak data...")
    path_bankrupt = Path(__file__).parent / "data/slovak_data/parsed_data/bankrupt/bankrupt_{}_{}_year_{}.csv" \
        .format(business_area, year, postfix)
    path_non_bankrupt = Path(__file__).parent / "data/slovak_data/parsed_data/non_bankrupt/nonbankrupt_{}_{}_year_{}" \
                                                ".csv".format(business_area, year, postfix)
    print("Data: {}".format(path_bankrupt))
    bankrupt_data = pd.read_csv(path_bankrupt)
    non_bankrupt_data = pd.read_csv(path_non_bankrupt)
    features = bankrupt_data.drop(["IS_BANKRUPT"], axis=1).append(non_bankrupt_data.drop(["IS_BANKRUPT"], axis=1))
    labels = bankrupt_data["IS_BANKRUPT"].append(non_bankrupt_data["IS_BANKRUPT"])
    print("Info: rows - {}, columns - {}".format(len(features), len(features.columns)))
    return features, labels

def get_scoring_dict():
    scoring_dict = {
        'accuracy_score': make_scorer(accuracy_score),
        'f1_score': make_scorer(f1_score),
        'roc_auc_score': make_scorer(roc_auc_score),
        'geometric_mean_score': make_scorer(geometric_mean_score),
        'sensitivity_score': make_scorer(sensitivity_score),
        'specificity_score': make_scorer(specificity_score)
    }
    return scoring_dict


import torch
from torch.optim.optimizer import Optimizer

class COCOB(Optimizer):
    r"""Implements COCOB algorithm.
    It has been proposed in `Training Deep Networks without Learning Rates Through Coin Betting`_.
    Arguments:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        alpha (float, optional): It was proposed to increase the stability in the first iterations,
            similarly and independently to the learning rate warm-up. The number roughly denotes the
            number of rounds of warm-up (default 100)
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
    .. _Training Deep Networks without Learning Rates Through Coin Betting:
        https://arxiv.org/abs/1705.07795
    """

    def __init__(self, params, alpha: float = 100, eps: float = 1e-8, weight_decay: float = 0):
        if not 0.0 <= eps:
            raise ValueError("Invalid epsilon value: {}".format(eps))
        if not 0.0 <= alpha:
            raise ValueError("Invalid alpha value: {}".format(alpha))
        if not 0.0 <= weight_decay:
            raise ValueError(
                "Invalid weight_decay value: {}".format(weight_decay))

        defaults = dict(weight_decay=weight_decay)
        self._alpha = alpha
        self._eps = eps

        super(COCOB, self).__init__(params, defaults)

    @torch.no_grad()
    def step(self, closure = None):
        """Performs a single optimization step.
        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        """
        loss = None
        if closure is not None:
            with torch.enable_grad():
                loss = closure()

        for group in self.param_groups:
            for p in group['params']:
                if p.grad is None:
                    continue
                grad = p.grad
                if grad.is_sparse:
                    raise RuntimeError('COCOB does not support sparse gradients')

                state = self.state[p]

                # State initialization
                if len(state) == 0:
                    # Sum of the negative gradients
                    state['sum_negative_gradients'] = torch.zeros_like(p).detach()
                    # Sum of the absolute values of the stochastic subgradients
                    state['grad_norm_sum'] = torch.zeros_like(p).detach()
                    # Maximum observed scale
                    state['L'] = self._eps*torch.ones_like(p).detach()
                    # Reward/wealth of the algorithm for each coordinate
                    state['reward'] = torch.zeros_like(p).detach()
                    # We need to save the initial point because this is a FTRL-based algorithm
                    state['x0'] = torch.clone(p.data).detach()

                sum_negative_gradients, grad_norm_sum, L, reward, x0 = (
                    state['sum_negative_gradients'],
                    state['grad_norm_sum'],
                    state['L'],
                    state['reward'],
                    state['x0'],
                )

                if group['weight_decay'] != 0:
                    grad = grad.add(p, alpha=group['weight_decay'])

                # update maximum rage of the gradients
                torch.max(L, torch.abs(grad), out=L)
                # udpate dual vector
                sum_negative_gradients.sub_(grad)
                # update sum of the absolute values of the gradients
                grad_norm_sum.add_(torch.abs(grad))
                # update the wealth
                reward.addcmul_(grad, p.data.sub(x0), value=-1)
                # reset the wealth to zero in case we lost all
                torch.maximum(reward, torch.zeros_like(reward), out=reward)
                # calculate denominator
                den = torch.maximum(grad_norm_sum.add(L), L.mul(self._alpha)).mul(L)
                # update model parameters
                p.data.copy_(reward.add(L).mul(sum_negative_gradients).div(den).add(x0))

        return loss


import re
import sys
from napalm import get_network_driver
from getpass import getpass

def pretty_print(d, indent=0):
    for key, value in d.items():
        print('\t' * indent + str(key))
        if isinstance(value, dict):
            pretty_print(value, indent+1)
        elif isinstance(value, list):
            for e in value:
                print('\t' * (indent+1) + str(e))
        else:
            print('\t' * (indent+1) + str(value))

def build_dict(cfg):
    """    Builds nested/deep dictionary from Cisco ios
    config using recursion function 'child()', which also
    changes "the most child" dictionaries to list if possible.
    For global Cisco commands make special keys based on
    first word in the command, e.g.: '# aaa #'
    """
    def group_global_childless(dct):
        for k in list(dct):
            if not dct[k]:
                dct.pop(k,None)
                w = k.split()
                if w[0] == 'no':
                    sec_name = f"# {w[1]} #"
                else:
                    sec_name = f"# {w[0]} #"
                if sec_name in dct.keys():
                    dct[sec_name].append(k)
                else:
                    dct.update({sec_name: [k]})

    def child(base_indent):
        nonlocal n
        result = {}
        while True:
            if n >= len(lines):
                break
            stripped = lines[n].lstrip()
            indent = len(lines[n]) - len(stripped)
            if base_indent >= indent:
                break
            n = n + 1
            result.update({stripped: child(indent)})
        # In case we got all values={} transform result to list
        if not [v for v in result.values() if v]:
            result = [k for k in result.keys()]
        return result

    n = 0
    cfg, special_cases = cut_special_cases(cfg)
    lines = cfg.splitlines()
    lines = [line for line in lines if line
                and not line.startswith('!')
                and not line.startswith('end')]
    dct = child(base_indent=-1)
    dct.update(special_cases)
    group_global_childless(dct)
    return(dct)

def cut_special_cases(cfg):
    """ Cut special cases (banners, boot markers) from config and
    put them in special_cases dictionary, that is also returned
    """
    special_cases = {}
    rgx = r"((?:(?P<type>(?:set\s+)*banner\s\w+\s+)(?P<delim>\S+))((.*\r?\n)+?.*?)(\3).*)"
    re_banners = re.findall(rgx,cfg)
    for r in re_banners:
        cfg = cfg.replace(r[0],"",1)
        special_cases.update({f"# {r[1]}#": r[0].splitlines()})
    rgx = r"boot-start-marker\r?\n(.*\r?\n)*boot-end-marker"
    re_boot = re.search(rgx,cfg)
    cfg = cfg.replace(re_boot[0],"",1)
    special_cases.update({"# boot #": re_boot[0].splitlines()})
    return cfg, special_cases

def main():
    """    Reads config from file passed as argument. Or connect
    to Cisco ios device asking interactively ip, user, password
    Then prints result with indentation
    """
    if len(sys.argv) >= 1:
        file_name = sys.argv[1]
        fp = open(file_name)
        content = fp.read()
        dct = build_dict(content)
    else:
        driver = get_network_driver('ios')
        ipaddress = input('IP address: ')
        username = input('Username: ')
        ios_conn = driver(ipaddress, username, getpass())
        ios_conn.open()
        cfgs = ios_conn.get_config()
        dct = build_dict(cfgs['running'])
    pretty_print(dct,1)

if __name__ == "__main__":
    main()


import multiprocessing
import optparse
import os
import re

from error import InvalidProjectGroupsError
from error import NoSuchProjectError
from error import RepoExitError
from event_log import EventLog
import progress

# Are we generating man-pages?
GENERATE_MANPAGES = os.environ.get("_REPO_GENERATE_MANPAGES_") == " indeed! "

# Number of projects to submit to a single worker process at a time.
# This number represents a tradeoff between the overhead of IPC and finer
# grained opportunity for parallelism. This particular value was chosen by
# iterating through powers of two until the overall performance no longer
# improved. The performance of this batch size is not a function of the
# number of cores on the system.
WORKER_BATCH_SIZE = 32

# How many jobs to run in parallel by default?  This assumes the jobs are
# largely I/O bound and do not hit the network.
DEFAULT_LOCAL_JOBS = min(os.cpu_count(), 8)

class UsageError(RepoExitError):
    """Exception thrown with invalid command usage."""

class Command:
    """Base class for any command line action in repo."""

    # Singleton for all commands to track overall repo command execution and
    # provide event summary to callers. Only used by sync subcommand currently.
    #
    # NB: This is being replaced by git trace2 events. See git_trace2_event_log.
    event_log = EventLog()

    # Whether this command is a "common" one, i.e. whether the user would
    # commonly use it or it's a more uncommon command. This is used by the help
    # command to show short-vs-full summaries.
    COMMON = False

    # Whether this command supports running in parallel. If greater than 0,
    # it is the number of parallel jobs to default to.
    PARALLEL_JOBS = None

    # Whether this command supports Multi-manifest. If False, then main.py will
    # iterate over the manifests and invoke the command once per (sub)manifest.
    # This is only checked after calling ValidateOptions, so that partially
    # migrated subcommands can set it to False.
    MULTI_MANIFEST_SUPPORT = True

    def __init__(
        self,
        repodir=None,
        client=None,
        manifest=None,
        git_event_log=None,
        outer_client=None,
        outer_manifest=None,
    ):
        self.repodir = repodir
        self.client = client
        self.outer_client = outer_client or client
        self.manifest = manifest
        self.git_event_log = git_event_log
        self.outer_manifest = outer_manifest

        # Cache for the OptionParser property.
        self._optparse = None

    def WantPager(self, _opt):
        return False

    def ReadEnvironmentOptions(self, opts):
        """Set options from environment variables."""

        env_options = self._RegisteredEnvironmentOptions()

        for env_key, opt_key in env_options.items():
            # Get the user-set option value if any
            opt_value = getattr(opts, opt_key)

            # If the value is set, it means the user has passed it as a command
            # line option, and we should use that. Otherwise we can try to set
            # it with the value from the corresponding environment variable.
            if opt_value is not None:
                continue

            env_value = os.environ.get(env_key)
            if env_value is not None:
                setattr(opts, opt_key, env_value)

        return opts

    @property
    def OptionParser(self):
        if self._optparse is None:
            try:
                me = "repo %s" % self.NAME
                usage = self.helpUsage.strip().replace("%prog", me)
            except AttributeError:
                usage = "repo %s" % self.NAME
            epilog = (
                "Run `repo help %s` to view the detailed manual." % self.NAME
            )
            self._optparse = optparse.OptionParser(usage=usage, epilog=epilog)
            self._CommonOptions(self._optparse)
            self._Options(self._optparse)
        return self._optparse

    def _CommonOptions(self, p, opt_v=True):
        """Initialize the option parser with common options.

        These will show up for *all* subcommands, so use sparingly.
        NB: Keep in sync with repo:InitParser().
        """
        g = p.add_option_group("Logging options")
        opts = ["-v"] if opt_v else []
        g.add_option(
            *opts,
            "--verbose",
            dest="output_mode",
            action="store_true",
            help="show all output",
        )
        g.add_option(
            "-q",
            "--quiet",
            dest="output_mode",
            action="store_false",
            help="only show errors",
        )

        if self.PARALLEL_JOBS is not None:
            default = "based on number of CPU cores"
            if not GENERATE_MANPAGES:
                # Only include active cpu count if we aren't generating man
                # pages.
                default = f"%default; {default}"
            p.add_option(
                "-j",
                "--jobs",
                type=int,
                default=self.PARALLEL_JOBS,
                help=f"number of jobs to run in parallel (default: {default})",
            )

        m = p.add_option_group("Multi-manifest options")
        m.add_option(
            "--outer-manifest",
            action="store_true",
            default=None,
            help="operate starting at the outermost manifest",
        )
        m.add_option(
            "--no-outer-manifest",
            dest="outer_manifest",
            action="store_false",
            help="do not operate on outer manifests",
        )
        m.add_option(
            "--this-manifest-only",
            action="store_true",
            default=None,
            help="only operate on this (sub)manifest",
        )
        m.add_option(
            "--no-this-manifest-only",
            "--all-manifests",
            dest="this_manifest_only",
            action="store_false",
            help="operate on this manifest and its submanifests",
        )

    def _Options(self, p):
        """Initialize the option parser with subcommand-specific options."""

    def _RegisteredEnvironmentOptions(self):
        """Get options that can be set from environment variables.

        Return a dictionary mapping environment variable name
        to option key name that it can override.

        Example: {'REPO_MY_OPTION': 'my_option'}

        Will allow the option with key value 'my_option' to be set
        from the value in the environment variable named 'REPO_MY_OPTION'.

        Note: This does not work properly for options that are explicitly
        set to None by the user, or options that are defined with a
        default value other than None.

        """
        return {}

    def Usage(self):
        """Display usage and terminate."""
        self.OptionParser.print_usage()
        raise UsageError()

    def CommonValidateOptions(self, opt, args):
        """Validate common options."""
        opt.quiet = opt.output_mode is False
        opt.verbose = opt.output_mode is True
        if opt.outer_manifest is None:
            # By default, treat multi-manifest instances as a single manifest
            # from the user's perspective.
            opt.outer_manifest = True

    def ValidateOptions(self, opt, args):
        """Validate the user options & arguments before executing.

        This is meant to help break the code up into logical steps. Some tips:
        * Use self.OptionParser.error to display CLI related errors.
        * Adjust opt member defaults as makes sense.
        * Adjust the args list, but do so inplace so the caller sees updates.
        * Try to avoid updating self state. Leave that to Execute.
        """

    def Execute(self, opt, args):
        """Perform the action, after option parsing is complete."""
        raise NotImplementedError

    @staticmethod
    def ExecuteInParallel(
        jobs, func, inputs, callback, output=None, ordered=False
    ):
        """Helper for managing parallel execution boiler plate.

        For subcommands that can easily split their work up.

        Args:
            jobs: How many parallel processes to use.
            func: The function to apply to each of the |inputs|. Usually a
                functools.partial for wrapping additional arguments. It will be
                run in a separate process, so it must be pickalable, so nested
                functions won't work. Methods on the subcommand Command class
                should work.
            inputs: The list of items to process. Must be a list.
            callback: The function to pass the results to for processing. It
                will be executed in the main thread and process the results of
                |func| as they become available. Thus it may be a local nested
                function. Its return value is passed back directly. It takes
                three arguments:
                - The processing pool (or None with one job).
                - The |output| argument.
                - An iterator for the results.
            output: An output manager. May be progress.Progess or
                color.Coloring.
            ordered: Whether the jobs should be processed in order.

        Returns:
            The |callback| function's results are returned.
        """
        try:
            # NB: Multiprocessing is heavy, so don't spin it up for one job.
            if len(inputs) == 1 or jobs == 1:
                return callback(None, output, (func(x) for x in inputs))
            else:
                with multiprocessing.Pool(jobs) as pool:
                    submit = pool.imap if ordered else pool.imap_unordered
                    return callback(
                        pool,
                        output,
                        submit(func, inputs, chunksize=WORKER_BATCH_SIZE),
                    )
        finally:
            if isinstance(output, progress.Progress):
                output.end()

    def _ResetPathToProjectMap(self, projects):
        self._by_path = {p.worktree: p for p in projects}

    def _UpdatePathToProjectMap(self, project):
        self._by_path[project.worktree] = project

    def _GetProjectByPath(self, manifest, path):
        project = None
        if os.path.exists(path):
            oldpath = None
            while path and path != oldpath and path != manifest.topdir:
                try:
                    project = self._by_path[path]
                    break
                except KeyError:
                    oldpath = path
                    path = os.path.dirname(path)
            if not project and path == manifest.topdir:
                try:
                    project = self._by_path[path]
                except KeyError:
                    pass
        else:
            try:
                project = self._by_path[path]
            except KeyError:
                pass
        return project

    def GetProjects(
        self,
        args,
        manifest=None,
        groups="",
        missing_ok=False,
        submodules_ok=False,
        all_manifests=False,
    ):
        """A list of projects that match the arguments.

        Args:
            args: a list of (case-insensitive) strings, projects to search for.
            manifest: an XmlManifest, the manifest to use, or None for default.
            groups: a string, the manifest groups in use.
            missing_ok: a boolean, whether to allow missing projects.
            submodules_ok: a boolean, whether to allow submodules.
            all_manifests: a boolean, if True then all manifests and
                submanifests are used. If False, then only the local
                (sub)manifest is used.

        Returns:
            A list of matching Project instances.
        """
        if all_manifests:
            if not manifest:
                manifest = self.manifest.outer_client
            all_projects_list = manifest.all_projects
        else:
            if not manifest:
                manifest = self.manifest
            all_projects_list = manifest.projects
        result = []

        if not groups:
            groups = manifest.GetGroupsStr()
        groups = [x for x in re.split(r"[,\s]+", groups) if x]

        if not args:
            derived_projects = {}
            for project in all_projects_list:
                if submodules_ok or project.sync_s:
                    derived_projects.update(
                        (p.name, p) for p in project.GetDerivedSubprojects()
                    )
            all_projects_list.extend(derived_projects.values())
            for project in all_projects_list:
                if (missing_ok or project.Exists) and project.MatchesGroups(
                    groups
                ):
                    result.append(project)
        else:
            self._ResetPathToProjectMap(all_projects_list)

            for arg in args:
                # We have to filter by manifest groups in case the requested
                # project is checked out multiple times or differently based on
                # them.
                projects = [
                    project
                    for project in manifest.GetProjectsWithName(
                        arg, all_manifests=all_manifests
                    )
                    if project.MatchesGroups(groups)
                ]

                if not projects:
                    path = os.path.abspath(arg).replace("\\", "/")
                    tree = manifest
                    if all_manifests:
                        # Look for the deepest matching submanifest.
                        for tree in reversed(list(manifest.all_manifests)):
                            if path.startswith(tree.topdir):
                                break
                    project = self._GetProjectByPath(tree, path)

                    # If it's not a derived project, update path->project
                    # mapping and search again, as arg might actually point to
                    # a derived subproject.
                    if (
                        project
                        and not project.Derived
                        and (submodules_ok or project.sync_s)
                    ):
                        search_again = False
                        for subproject in project.GetDerivedSubprojects():
                            self._UpdatePathToProjectMap(subproject)
                            search_again = True
                        if search_again:
                            project = (
                                self._GetProjectByPath(manifest, path)
                                or project
                            )

                    if project:
                        projects = [project]

                if not projects:
                    raise NoSuchProjectError(arg)

                for project in projects:
                    if not missing_ok and not project.Exists:
                        raise NoSuchProjectError(
                            "%s (%s)"
                            % (arg, project.RelPath(local=not all_manifests))
                        )
                    if not project.MatchesGroups(groups):
                        raise InvalidProjectGroupsError(arg)

                result.extend(projects)

        def _getpath(x):
            return x.relpath

        result.sort(key=_getpath)
        return result

    def FindProjects(self, args, inverse=False, all_manifests=False):
        """Find projects from command line arguments.

        Args:
            args: a list of (case-insensitive) strings, projects to search for.
            inverse: a boolean, if True, then projects not matching any |args|
                are returned.
            all_manifests: a boolean, if True then all manifests and
                submanifests are used. If False, then only the local
                (sub)manifest is used.
        """
        result = []
        patterns = [re.compile(r"%s" % a, re.IGNORECASE) for a in args]
        for project in self.GetProjects("", all_manifests=all_manifests):
            paths = [project.name, project.RelPath(local=not all_manifests)]
            for pattern in patterns:
                match = any(pattern.search(x) for x in paths)
                if not inverse and match:
                    result.append(project)
                    break
                if inverse and match:
                    break
            else:
                if inverse:
                    result.append(project)
        result.sort(
            key=lambda project: (project.manifest.path_prefix, project.relpath)
        )
        return result

    def ManifestList(self, opt):
        """Yields all of the manifests to traverse.

        Args:
            opt: The command options.
        """
        top = self.outer_manifest
        if not opt.outer_manifest or opt.this_manifest_only:
            top = self.manifest
        yield top
        if not opt.this_manifest_only:
            yield from top.all_children

class InteractiveCommand(Command):
    """Command which requires user interaction on the tty and must not run
    within a pager, even if the user asks to.
    """

    def WantPager(self, _opt):
        return False

class PagedCommand(Command):
    """Command which defaults to output in a pager, as its display tends to be
    larger than one screen full.
    """

    def WantPager(self, _opt):
        return True

class MirrorSafeCommand:
    """Command permits itself to run within a mirror, and does not require a
    working directory.
    """

class GitcClientCommand:
    """Command that requires the local client to be a GITC client."""


# data_processing.py
from shapely import wkb
from shapely.geometry import shape
import binascii
import psycopg2

ewkb_data = None

def get_variable_ewkb():
    global ewkb_data
    print("variable ewkb")
    print(ewkb_data)

    return ewkb_data

def process_for_view(data):
    global ewkb_data  # Declare ewkb_data as a global variable
    # Process the data as needed in your Python program
    # For example, you can access the features using data['features']
    # and perform further processing or analysis
    print("Processing data...")
    geojson_data = data

    # Convert GeoJSON to Shapely geometry
    shapely_geometry = shape(geojson_data['geometry'])

    # Convert Shapely geometry to WKB (Well-Known Binary) format
    wkb_data = shapely_geometry.wkb

    # Convert bytes to hexadecimal string
    wkb_hex = wkb_data.hex()

    # Convert Shapely geometry to EWKB (Extended Well-Known Binary) format
    ewkb_data = wkb.dumps(shapely_geometry, srid=4326, hex=True)

    print(wkb_hex)
    print(ewkb_data)

    return ewkb_data


import serial
from sense_hat import SenseHat
from socket import gethostname
from xbee import XBee
from statistics import median

def clear_matrix():
    sense.clear()

def show_hostname():
    hostname = gethostname()
    sense.show_message("Hostname: " + hostname)

def receive_data(data):
    print("received data: ", data)
    if (data["rf_data"] == b"ping"):
        # Received ping? -> Send pong back
        send_data("pong")
    elif (data["rf_data"] == b"pong"):
        # Received pong? -> Store & calc RSSI
        rssi_list.append(ord(data["rssi"]))
        print_current_rssi_median()

        if data["source_addr"] not in rssi_list:
            rssi_dict[data["source_addr"]] = []
        rssi_dict[data["source_addr"]].append(ord(data["rssi"]))

def send_data(data, dest_addr="\x00\x0A"):
    xbee.send("tx",
              frame_id="\x00",
              dest_addr=dest_addr,
              data=data)

def init_rssi_calc(n_pings=10):
    rssi_list = []
    dest_addr = "\x00\x0A" # 2byte hex value (TODO: Set according to adress of destination XBee module)
    for i in range(n_pings):
        send_data("ping", dest_addr)

def print_current_rssi_median():
    med = median(rssi_list)
    print("Current RSSI median with {} received pongs: {}".format(len(rssi_list), med))
    dist = dist_from_rssi(med)
    print("Current dist from RSSI: dist = {}".format(dist))

def dist_from_rssi(rssi):
    n = 2.8 # path loss variable from 2 to 4
    A = 33 # TODO: Measure reference RSSI (1m distance)
    # RSSI = -10 * n * log_10(d) + A
    # => Transformed to
    # d = 10^(A/10n)
    dist = 10**(A/10*n)
    return dist

def three_anchor_bbox():
    dist_dict = {}
    for anchor in rssi_dict:
        dist_dict[anchor] = dist_from_rssi(median(rssi_dict[anchor]))

    x = 1/2 * (min([anchor_positions[anchor][0] + dist_dict[anchor] for anchor in anchor_positions]) + \
        max([anchor_positions[anchor][0] - dist_dict[anchor] for anchor in anchor_positions]))

    y = 1/2 * (min([anchor_positions[anchor][1] + dist_dict[anchor] for anchor in anchor_positions]) + \
        max([anchor_positions[anchor][1] - dist_dict[anchor] for anchor in anchor_positions]))

    print(">> BBox calculation done: X: {} | Y: {}".format(x,y))

def three_anchor_multilat():
    print("... Starting multilateration.")
    dist_dict = {}
    for anchor in rssi_dict:
        dist_dict[anchor] = dist_from_rssi(median(rssi_dict[anchor]))

    # https://github.com/kamalshadi/Localization
    import localization as lx
    P=lx.Project(mode='2D',solver='LSE')

    print("... adding anchors")
    for anchor in anchor_positions:
        P.add_anchor(anchor, anchor_positions[anchor])

    t,label = P.add_target()

    print("... adding measurements")
    for dist in dist_dict:
        P.add_measure(dist, dist_dict[dist])

    print("... calculating...")
    P.solve()

    print("> Done! Multilat result:", t.loc)

def broadcast_ping():
    rssi_dict = {}
    # Broadcast ping
    send_data("ping", "\xFF\xFF")

if __name__ == "__main__":
    sense = SenseHat()
    print(">> Opening serial port...")
    ser = serial.Serial("/dev/ttyUSB1", 9600)
    xbee = XBee(ser, callback=receive_data)

    rssi_list = []
    rssi_dict = {}

    ## TODO: Fill anchor position dictionary!
    anchor_positions = {
        "add1": (0, 1),
        "add2": (0, 3),
        "add3": (0, 4)
    }

    print(">> Waiting for events...")
    print("Middle: clear_matrix, left: init_rssi_calc, right: three_anchor_bbox, down: broadcast_ping")
    print("Sequence: broadcast_ping -> three_anchor_bbox / three_anchor_multilat")
    while True:
        for event in sense.stick.get_events():
            if event.action == "pressed":
                if event.direction == "middle":
                    print("** Event: Pressed.")
                    clear_matrix()
                elif event.direction == "left":
                    print("** Event: Left")
                    init_rssi_calc()
                elif event.direction == "right":
                    print("** Event: Right.")
                    three_anchor_bbox()
                elif event.direction == "down":
                    print("** Event: Down")
                    broadcast_ping()
                elif event.direction == "up":
                    print("** Event: up")
                    three_anchor_multilat()