"""Pylons application test package

This package assumes the Pylons environment is already loaded, such as
when this script is imported from the `nosetests --with-pylons=test.ini`
command.

This module initializes the application via ``websetup`` (`paster
setup-app`) and provides the base testing objects.
"""
from unittest import TestCase
import datetime
import sys

from hamcrest import assert_that, is_, contains_string, is_not, any_of, not_none
from mock import Mock
import os
import pylons
from pylons.i18n.translation import _get_translator
from paste.deploy import loadapp
from pylons import url
import re
from routes.util import URLGenerator
from sqlalchemy import or_
from webtest import TestApp

from joj.model import User, Dataset, ParameterValue, ModelRunStatus, \
    DrivingDatasetParameterValue, DrivingDataset, DrivingDatasetLocation, SystemAlertEmail, \
    AccountRequest, LandCoverAction, LandCoverRegion, LandCoverRegionCategory
from joj.services.user import UserService
from joj.utils import constants, f90_helper
from joj.services.model_run_service import ModelRunService
from joj.model import session_scope, Session, ModelRun
from joj.services.dap_client.dap_client_factory import DapClientFactory
from joj.model.non_database.driving_dataset_jules_params import DrivingDatasetJulesParams

TEST_LOG_FORMAT_STRING = '%(name)-20s %(asctime)s ln:%(lineno)-3s %(levelname)-8s\n %(message)s\n'

__all__ = ['environ', 'url', 'TestController']

environ = {}
here_dir = os.path.dirname(os.path.abspath(__file__))
conf_dir = os.path.dirname(os.path.dirname(here_dir))

sys.path.insert(0, conf_dir)

class TestController(TestCase):
    def __init__(self, *args, **kwargs):
        wsgiapp = loadapp('config:test.ini', relative_to=conf_dir)
        config = wsgiapp.config
        pylons.app_globals._push_object(config['pylons.app_globals'])
        pylons.config._push_object(config)

        # Initialize a translator for tests that utilize i18n
        translator = _get_translator(pylons.config.get('lang'))
        pylons.translator._push_object(translator)

        url._push_object(URLGenerator(config['routes.map'], environ))
        self.app = TestApp(wsgiapp)
        TestCase.__init__(self, *args, **kwargs)

    def login(self,
              username="test",
              access_level=constants.USER_ACCESS_LEVEL_EXTERNAL,
              workbench_username=None):
        """
        Setup the request as if the user has already logged in as a non admin user

        :param username: the username for the user to log in, stored in self.login_username, default "test"
        :param access_level: the access level of the user
        :param workbench_username: the workbench username for this user
        :return the details for the logged in user
        """
        self.login_username = username
        user_service = UserService()
        user = user_service.get_user_by_username(self.login_username)
        if user is None:
            user_service.create(
                self.login_username,
                'test',
                'testerson',
                'test@ceh.ac.uk',
                access_level,
                workbench_username=workbench_username)
            user = user_service.get_user_by_username(self.login_username)

        self.app.extra_environ['REMOTE_USER'] = str(user.username)
        return user

    def clean_database(self):
        """
        Cleans the User, ModelRun, Dataset, DrivingDataset and ParameterValue tables in the database
        """
        with session_scope(Session) as session:

            published_status = session \
                .query(ModelRunStatus) \
                .filter(ModelRunStatus.name == constants.MODEL_RUN_STATUS_PUBLISHED) \
                .one()

            parameter_to_keep = session \
                .query(ParameterValue.id) \
                .join(ModelRun) \
                .join(User) \
                .filter(User.username == constants.CORE_USERNAME) \
                .filter(ModelRun.status_id != published_status.id)\
                .all()

            session.query(LandCoverAction).delete()
            session.query(LandCoverRegion).delete()
            session.query(LandCoverRegionCategory).delete()

            session \
                .query(ParameterValue) \
                .filter(ParameterValue.id.notin_([x[0] for x in parameter_to_keep])) \
                .delete(synchronize_session='fetch')

            core_user_id = session.query(User.id).filter(User.username == constants.CORE_USERNAME).one()[0]

            session.query(Dataset).delete()

            # delete all runs except the scientific configurations
            session \
                .query(ModelRun) \
                .filter(or_(
                ModelRun.user_id != core_user_id,
                ModelRun.user_id.is_(None),
                ModelRun.status_id == published_status.id)) \
                .delete(synchronize_session='fetch')

            session.query(DrivingDatasetLocation).delete()
            session.query(DrivingDatasetParameterValue).delete()
            session.query(DrivingDataset) \
                .filter(DrivingDataset.name != constants.USER_UPLOAD_DRIVING_DATASET_NAME) \
                .delete()

            session \
                .query(User) \
                .filter(or_(User.username != constants.CORE_USERNAME, User.username.is_(None))) \
                .delete()

            session.query(AccountRequest).delete()

            emails = session \
                .query(SystemAlertEmail).all()
            for email in emails:
                email.last_sent = None

    def assert_model_definition(self, expected_username, expected_science_configuration, expected_name,
                                expected_description):
        """
        Check that a model definition is correct in the database. Throws assertion error if there is no match

        Arguments:
        expected_name -- the expected name
        expected_science_configuration -- the expected science configuration id
        """
        session = Session()
        row = session.query("username", "name", "science_configuration_id", "description").from_statement(
            """
            SELECT u.username, m.name, m.science_configuration_id, m.description
            FROM model_runs m
            JOIN model_run_statuses s on s.id = m.status_id
            JOIN users u on u.id = m.user_id
            WHERE s.name=:status
              AND u.username = :username
            """
        ).params(status=constants.MODEL_RUN_STATUS_CREATED, username=expected_username).one()
        assert_that(row.username, is_(expected_username), "username")
        assert_that(row.name, is_(expected_name), "model run name")
        assert_that(row.science_configuration_id, is_(expected_science_configuration), "science_configuration_id")
        assert_that(row.description, is_(expected_description), "description")

    def assert_parameter_of_model_being_created_is_a_value(self, parameter_id, expected_parameter_value):
        """
        Assert that the parameter value is correct for the model being created
        :param parameter_id: the id of the parameter to assert the value of
        :param expected_parameter_value: expected parameters value
        :return:Nothing
        """

        session = Session()
        row = session.query("value").from_statement(
            """
            SELECT pv.value
            FROM model_runs m
            JOIN model_run_statuses s on s.id = m.status_id
            JOIN parameter_values pv on pv.model_run_id = m.id
            JOIN parameters p on pv.parameter_id = p.id
            WHERE s.name=:status
              AND p.id = :parameter_id
            """
        ).params(status=constants.MODEL_RUN_STATUS_CREATED, parameter_id=parameter_id).one()
        assert_that(row.value, is_(expected_parameter_value), "parameter value")

    def _status(self, status_name):
        """
        Return from the database the ModelRunStatus for a requested status name
        :param status_name: The name of the status to find
        :return: The matching status object
        """
        with session_scope(Session) as session:
            return session.query(ModelRunStatus).filter(ModelRunStatus.name == status_name).one()

    def create_driving_dataset(
            self,
            session,
            jules_params=DrivingDatasetJulesParams(dataperiod=3600, var_interps=8 * ["i"]),
            is_restricted_to_admins=False,
            data_range_from = 10,
            data_range_to = 5):
        """
        Create a driving dataset
        :param session: session to use
        :param jules_params: set of jules parameters
        :return: dataset
        """
        model_run_service = ModelRunService()

        driving1 = DrivingDataset()
        driving1.name = "driving1"
        driving1.description = "driving 1 description"
        driving1.geographic_region = 'European'
        driving1.spatial_resolution = '1km'
        driving1.temporal_resolution = '24 hours'
        driving1.boundary_lat_north = 50
        driving1.boundary_lat_south = -10
        driving1.boundary_lon_west = -15
        driving1.boundary_lon_east = 30
        driving1.time_start = datetime.datetime(1979, 1, 1, 0, 0, 0)
        driving1.time_end = datetime.datetime(2010, 1, 1, 0, 0, 0)
        driving1.view_order_index = 100
        driving1.is_restricted_to_admins = is_restricted_to_admins
        location1 = DrivingDatasetLocation()
        location1.base_url = "base_url"
        location1.dataset_type_id = 1
        location1.driving_dataset = driving1
        location2 = DrivingDatasetLocation()
        location2.base_url = "base_url2"
        location2.driving_dataset = driving1
        location2.dataset_type_id = 1
        jules_params.add_to_driving_dataset(model_run_service, driving1, session)

        val = f90_helper.python_to_f90_str(8 * ["i"])
        pv1 = DrivingDatasetParameterValue(model_run_service, driving1,
                                           constants.JULES_PARAM_DRIVE_INTERP, val)
        val = f90_helper.python_to_f90_str(3600)
        pv2 = DrivingDatasetParameterValue(model_run_service, driving1,
                                           constants.JULES_PARAM_DRIVE_DATA_PERIOD, val)
        val = f90_helper.python_to_f90_str("data/driving1/frac.nc")
        pv3 = DrivingDatasetParameterValue(model_run_service, driving1,
                                           constants.JULES_PARAM_FRAC_FILE, val)
        val = f90_helper.python_to_f90_str("frac")
        pv4 = DrivingDatasetParameterValue(model_run_service, driving1,
                                           constants.JULES_PARAM_FRAC_NAME, val)

        session.add(driving1)
        session.commit()

        driving_data_filename_param_val = DrivingDatasetParameterValue(
            model_run_service,
            driving1,
            constants.JULES_PARAM_DRIVE_FILE,
            "'testFileName'")
        session.add(driving_data_filename_param_val)
        session.commit()

        return driving1

    def create_two_driving_datasets(self):
        """
        Creates two driving datasets with datasets, parameter values etc set up
        :return: nothing
        """

        with session_scope(Session) as session:
            self.create_driving_dataset(session)

            model_run_service = ModelRunService()

            driving2 = DrivingDataset()
            driving2.name = "driving2"
            driving2.description = "driving 2 description"
            driving2.geographic_region = 'Global'
            driving2.spatial_resolution = 'Half degree'
            driving2.temporal_resolution = '3 Hours'
            driving2.boundary_lat_north = 85
            driving2.boundary_lat_south = -90
            driving2.boundary_lon_west = -180
            driving2.boundary_lon_east = 180
            driving2.time_start = datetime.datetime(1901, 1, 1, 0, 0, 0)
            driving2.time_end = datetime.datetime(2001, 1, 1, 0, 0, 0)
            driving2.view_order_index = 200
            driving2.usage_order_index = 2
            driving2.is_restricted_to_admins = False

            location3 = DrivingDatasetLocation()
            location3.base_url = "base_url3"
            location3.driving_dataset = driving2

            val = f90_helper.python_to_f90_str(8 * ["i"])
            pv1 = DrivingDatasetParameterValue(model_run_service, driving2,
                                               constants.JULES_PARAM_DRIVE_INTERP, val)
            val = f90_helper.python_to_f90_str(3600)
            pv2 = DrivingDatasetParameterValue(model_run_service, driving2,
                                               constants.JULES_PARAM_DRIVE_DATA_PERIOD, val)
            val = f90_helper.python_to_f90_str("data/driving2/frac.nc")
            pv3 = DrivingDatasetParameterValue(model_run_service, driving2,
                                               constants.JULES_PARAM_FRAC_FILE, val)
            val = f90_helper.python_to_f90_str("frac2")
            pv4 = DrivingDatasetParameterValue(model_run_service, driving2,
                                               constants.JULES_PARAM_FRAC_NAME, val)
            val = f90_helper.python_to_f90_str(['b', 'sathh', 'satcon', 'sm_sat', 'sm_crit',
                                                'sm_wilt', 'hcap', 'hcon', 'albsoil'])
            pv5 = DrivingDatasetParameterValue(model_run_service, driving2,
                                               constants.JULES_PARAM_SOIL_PROPS_VAR, val)
            val = f90_helper.python_to_f90_str(['bexp', 'sathh', 'satcon', 'vsat', 'vcrit',
                                                'vwilt', 'hcap', 'hcon', 'albsoil'])
            pv6 = DrivingDatasetParameterValue(model_run_service, driving2,
                                               constants.JULES_PARAM_SOIL_PROPS_VAR_NAME, val)
            val = f90_helper.python_to_f90_str(9)
            pv7 = DrivingDatasetParameterValue(model_run_service, driving2,
                                               constants.JULES_PARAM_SOIL_PROPS_NVARS, val)
            val = f90_helper.python_to_f90_str("data/WATCH_2D/ancils/soil_igbp_bc_watch_0p5deg_capUM6.6_2D.nc")
            pv7 = DrivingDatasetParameterValue(model_run_service, driving2,
                                               constants.JULES_PARAM_SOIL_PROPS_FILE, val)
            session.add(driving2)
            session.commit()

    def create_dataset(self, session, data_range_from=40, data_range_to=1000):
        """
        Create a standard dataset
        :param session: session to sue to create the dataset
        :param data_range_from: data range from
        :param data_range_to: data range to
        :return:
        """

        ds = Dataset()
        ds.data_range_from = data_range_from
        ds.data_range_to = data_range_to
        session.add(ds)
        session.flush()
        return ds.id

    def assert_model_run_status_and_return(self, model_run_id, status):
        """
        assert that a model has a given status
        :param model_run_id: id of the model
        :param status: name of the status
        :return:the model run
        """
        with session_scope(Session) as session:
            result = session \
                .query(ModelRun) \
                .filter(ModelRun.id == model_run_id) \
                .one()
            assert_that(result.status.name, is_(status))
            return result

    def create_run_model(self, storage_in_mb, name, user, status=constants.MODEL_RUN_STATUS_COMPLETED):
        """
        Create a model run
        :param storage_in_mb: storage_in_mb for the model
        :param name: name of the model run
        :param user: user who has created the model run
        :param status: the status, default to complete
        :return:the model run
        """
        model_run_service = ModelRunService()
        with model_run_service.transaction_scope() as session:
            model_run = model_run_service._create_new_model_run(session, user)
            session.add(model_run)
            model_run.name = name
            science_configuration = model_run_service._get_science_configuration(
                constants.DEFAULT_SCIENCE_CONFIGURATION, session)
            model_run.science_configuration_id = science_configuration.id
            model_run.code_version = science_configuration.code_version
            model_run.description = "testing"
            model_run_service._copy_parameter_set_into_model(science_configuration.parameter_values, model_run, session)
            model_run.storage_in_mb = storage_in_mb
            model_run.change_status(session, status)

        return model_run

    def assert_model_run_creation_action(self, user, expected_action):
        """
        Assert that the user model run creation expected_action is correct
        :param user: user
        :param expected_action: expected_action
        :return:nothing
        """
        with session_scope(Session) as session:
            modified_user = session.query(User).filter(User.id == user.id).one()
            assert_that(modified_user.model_run_creation_action, is_(expected_action), "model run creation action")

    def create_mock_dap_factory_client(self, data_range_from=10, data_range_to=12, longname="long_name"):
        """
        Create a dap factory which returns a mock dap client
        :param data_range_to: maximum data value for dap client to return
        :param data_range_from: minimum data value for dap client to return
        :param longname: long name that client should return
        :return:
        """
        self.dap_client = Mock()
        self.dap_client.get_longname = Mock(return_value=longname)
        self.dap_client.get_data_range = Mock(return_value=[data_range_from, data_range_to])
        self.dap_client_factory = DapClientFactory()
        self.dap_client_factory.get_dap_client = Mock(return_value=self.dap_client)
        return self.dap_client_factory

    def create_account_request(self, email_addess="test@test.ceh.ac.uk"):
        """
        Create an account request
        :return: id of account request
        """
        with session_scope() as session:
            self.account_request = AccountRequest()
            self.account_request.email = email_addess
            self.account_request.institution = "institution"
            self.account_request.first_name = "a very unique name for the user " + str(datetime.datetime.now())
            self.account_request.last_name = "last name"
            self.account_request.usage = "usage"
            session.add(self.account_request)

        with session_scope() as session:
            ac = session.query(AccountRequest).filter(
                AccountRequest.first_name == self.account_request.first_name).one()
            return ac.id

    def add_land_cover_actions(self, land_cover_region, model_run, value_order_pairs, land_cover_service):
        """
        Create land cover actions and save them using the model_run_service.save_land_cover_actions method
        :param land_cover_region: Land cover region actions should belong to
        :param model_run: Model run to add them against
        :param value_order_pairs: List of 2-tuples [(value, order)]; each tuple is a land cover action to be added
        :param land_cover_service: Land Cover service to use
        :return:
        """
        land_cover_actions = []
        for value, order in value_order_pairs:
            lca = LandCoverAction()
            lca.region_id = land_cover_region.id
            lca.value_id = value
            lca.order = order
            land_cover_actions.append(lca)
        land_cover_service.save_land_cover_actions_for_model(model_run, land_cover_actions)

    def add_land_cover_region(self, model_run):
        with session_scope() as session:
            land_cover_cat = LandCoverRegionCategory()
            land_cover_cat.name = "Countries"
            land_cover_cat.driving_dataset_id = model_run.driving_dataset_id

            land_cover_region = LandCoverRegion()
            land_cover_region.mask_file = "filepath"
            land_cover_region.name = "Wales"
            land_cover_region.category = land_cover_cat

            session.add(land_cover_region)
        return land_cover_region

    def _add_model_run_being_created(self, user):
        model_run_service = ModelRunService()
        model_run_service.update_model_run(user, "test", constants.DEFAULT_SCIENCE_CONFIGURATION)

    def assert_input_has_correct_label_and_value(self, element_id, label, response, value):
        assert_that(response.normal_body, contains_string(label))
        workbench_input = re.search('<input[^>]* id="{}"[^>]*>'.format(element_id), response.normal_body)
        assert_that(workbench_input, not_none(), "input file exists with id of {}".format(element_id))
        if value is None or value is "":
            assert_that(workbench_input.group(0), any_of(
                contains_string('value=""'.format(value)),
                is_not(contains_string('value="'))
                ))
        else:
            assert_that(workbench_input.group(0), contains_string('value="{}"'.format(value)))


import abc
import optparse
import random
import numpy as np
from typing import cast
from scipy import signal
from scipy import fft
from scipy import interpolate
from matplotlib import colormaps
from math import ceil
from pytui import StyledWindow, Plot, Text
from radio import Radio, RtlRadio, PlutoRadio

# returns a matplotlib color map as rgb tuples
def get_colormap(name: str) -> list[tuple[int, int, int]]:
    colors = [colormaps[name](x) for x in range(0, 256)]
    return [(int(r*255), int(g*255), int(b*255)) for (r, g, b, a) in colors]

# returns a gradient between two colors as rgb tuples
def color_gradient(
    start: tuple[int, int, int],
    end: tuple[int, int, int],
    n: int
) -> list[tuple[int, int, int]]:
    gradient = [start]
    for t in range(1, n):
        gradient.append(cast(tuple[int, int, int], tuple([
          int(start[j] + (float(t)/(n-1))*(end[j]-start[j]))
          for j in range(3)
        ])))
    return gradient

def get_matrix_colormap() -> list[tuple[int, int, int]]:
    black = (0, 0, 0)
    vampire = (0x0d, 0x02, 0x08)
    dark_green = (0x00, 0x3b, 0x00)
    islamic_green = (0x00, 0x8f, 0x11)
    malachite = (0x00, 0xff, 0x41)
    return (
        color_gradient(black, vampire, 96)
        + color_gradient(vampire, dark_green, 32)
        + color_gradient(dark_green, islamic_green, 64)
        + color_gradient(islamic_green, malachite, 64)
    )

Styles: dict[str, dict] = {
    # colors from
    # https://marketplace.visualstudio.com/items?itemName=enkia.tokyo-night
    'tokyonight': {
        'waterfall-glyph': lambda: '█',
        'colormap': get_colormap('viridis'),
        'header': {'bg': 0x1a1b26, 'fg': 0xa9b1d6},
        # 'plot': {'bg': 0x24283b, 'fg': 0xf7768e},   # red
        'plot': {'bg': 0x24283b, 'fg': 0x7aa2f7},   # blue
        'plot-label': {'bg': 0x24283b, 'fg': 0x565f89}
    },

    # https://matplotlib.org/matplotblog/posts/matplotlib-cyberpunk-style/
    'cyberpunk': {
        'waterfall-glyph': lambda: '█',
        'colormap': get_colormap('viridis'),
        'header': {'bg': 0x2A3459, 'fg': 0x08F7FE},
        # 'plot': {'bg': 0x2A3459, 'fg': 0x08F7FE},
        'plot': {'bg': 0x2A3459, 'fg': 0xFE53BB},
        'plot-label': {'bg': 0x2A3459, 'fg': 0x08F7FE}
    },

    # https://www.schemecolor.com/matrix-code-green.php
    'matrix': {
        'waterfall-glyph': lambda: random.choice(
            'ｦｱｲｳｴｵｶｷｸｹｺｻｼｽｾｿﾀﾁﾂﾃﾄﾅﾆﾇﾈﾉﾊﾋﾌﾍﾎﾏﾐﾑﾒﾓﾔﾕﾖﾗﾘﾙﾚﾛﾜﾝ'
        ),
        'colormap': get_matrix_colormap(),
        'header': {'bg': 0x0d0208, 'fg': 0x008f11},
        'plot': {'bg': 0x0d0208, 'fg': 0x00ff41},
        'plot-label': {'bg': 0x0d0208, 'fg': 0x008f11}
    },

    # https://www.nordtheme.com/docs/colors-and-palettes
    'nord': {
        'waterfall-glyph': lambda: '█',
        'colormap': get_colormap('cividis'),
        'header': {'bg': 0xd8dee9, 'fg': 0x81a1c1},
        'plot': {'bg': 0xeceff4, 'fg': 0x2e3440},
        'plot-label': {'bg': 0xd8dee9, 'fg': 0x81a1c1}
    },

}

def format_freq(f: float) -> str:
    if f >= 1e9:
        (f, s) = (f/1e6, 'Ghz')
    else:
        (f, s) = (f/1e3, 'Mhz')

    return f'{(f/1000):.4f}'.rstrip('0').rstrip('.') + ' ' + s

def make_frequency_labels(width: int, start: float, end: float) -> str:
    # *2 to account for flipping at midpoint
    maxWidth = 2*len('|999.9999 Mhz ')
    n = ceil(width/maxWidth)
    # ensure odd number of ticks to get one in the middle
    n = n-1 if n % 2 == 0 else n

    ticks = np.linspace(0, width, n)
    values = np.linspace(start, end, len(ticks))
    labels = ''.ljust(width)

    for i in range(0, n):
        hz = format_freq(values[i])
        if i < n / 2:
            (hz, x) = ('|'+hz, int(ticks[i]))
        else:
            (hz, x) = (hz+'|', int(ticks[i]) - len(hz+'|'))
        labels = labels[:x] + hz + labels[x + len(hz):]

    return labels

def make_db_labels(width: int, height: int, mindb: float, maxdb: float) -> str:
    scale = [' '.ljust(width)]*height
    labels = np.linspace(maxdb, mindb, height//2)
    for i, label in enumerate(labels):
        scale[i*2] = str(round(label)).rjust(width)
    return "\n".join(scale)

def get_psd(
    sample: list[np.complex64],
    window: str = 'hann',
    nperseg: int = None,
    fs: int = None
) -> list[float]:
    if nperseg is None or nperseg == len(sample):
        _, power = signal.periodogram(
            sample,
            window=window,
            return_onesided=False
        )
    else:
        _, power = signal.welch(
            sample,
            window=window,
            return_onesided=False,
            nperseg=nperseg,
            fs=fs
        )
    return fft.fftshift(power)

def to_db(values: list[float]) -> list[float]:
    return list(10.0 * np.log10(values))

def zero_adjust(values: list[float], offset: float) -> list[float]:
    return [x - offset for x in values]

def to_dbfs(values: list[float], offset: float) -> list[float]:
    return zero_adjust(to_db(values), offset)

# fits a list of one size to another by lerping the values
def fit_values(values: list[float], length: int) -> list[float]:
    if len(values) == length:
        return values
    # create function to interpolate from 0-length to values
    x = np.arange(0, len(values))
    f = interpolate.interp1d(np.arange(0, len(values)), values, kind='nearest')
    # linearly subdivide x into length sample
    sx = np.linspace(x[0], x[-1], length)
    # scale sample into new length
    return f(sx)

class Visualizer(abc.ABC):
    def __init__(
        self,
        radio: Radio,
        style: dict = Styles['tokyonight']
    ) -> None:
        self.radio = radio
        self.windows: list[StyledWindow] = []
        self.style = style

    @abc.abstractmethod
    def layout(self, container: StyledWindow) -> None:
        pass

    @abc.abstractmethod
    def update_sample(self, sample: list[np.complex64]) -> None:
        pass

    @abc.abstractmethod
    def update_radio(self, radio: Radio) -> None:
        pass

    def draw(self) -> None:
        for window in self.windows:
            window.draw()

class FFTVisualizer(Visualizer):
    def __init__(
        self,
        radio: Radio,
        mindbfs: int = 0,
        maxdbfs: int = 50,
        nperseg: int = 1024,
        window: str = 'hann',
        zoff: float = -1.0,
        style: dict = Styles['tokyonight']
    ) -> None:
        super().__init__(radio, style)
        self.mindbfs = mindbfs
        self.maxdbfs = maxdbfs
        self.nperseg = nperseg
        self.window = window
        self.zoff = zoff

    def get_dbfs(self, sample: list[np.complex64]) -> list[float]:
        return to_dbfs(get_psd(
            sample,
            window=self.window,
            nperseg=self.nperseg,
            fs=self.radio.rx_bw()
        ), self.zoff)

class Seek(FFTVisualizer):
    def __init__(
        self,
        radio: Radio,
        fstart: float,
        fstop: float,
        mindbfs: int = 0,
        maxdbfs: int = 50,
        nperseg: int = 1024,
        window: str = 'hann',
        zoff: float = -1.0,
        style: dict = Styles['tokyonight']
    ) -> None:
        super().__init__(radio, mindbfs, maxdbfs, nperseg, window, zoff, style)
        self.fstart = fstart
        self.fstop = fstop
        self.signals: dict[int, float] = {}

    def find_signals(
        self,
        sample: list[np.complex64]
    ) -> list[tuple[int, float]]:
        dbfs = self.get_dbfs(sample)
        return [(i, x) for i, x in enumerate(dbfs) if x >= self.mindbfs]

    def have_signal(self, sample: list[np.complex64]) -> bool:
        return len(self.find_signals(sample)) > 0

    def layout(self, container: StyledWindow) -> None:
        dbwidth = max(len(str(self.mindbfs)), len(str(self.maxdbfs)))

        (self.header, body) = container.hsplit(1)
        (self.yaxis, body) = body.vsplit(dbwidth)
        (self.plot, self.xaxis) = body.hsplit(None, 1)

        self.windows = [self.header, self.plot, self.xaxis, self.yaxis]

        self.header.update_style(self.style['header'])
        self.plot.update_style(self.style['plot'])
        self.xaxis.update_style(self.style['plot-label'])
        self.yaxis.update_style(self.style['plot-label'])

        self.update_header()
        self.update_yaxis()
        self.update_xaxis()

    def update_header(self) -> None:
        f = self.radio.rx_freq()
        peak = 'None'
        if self.signals:
            peakf = max(self.signals, key=lambda k: self.signals[k])
            peakdb = round(self.signals[peakf])
            peak = f'{peakf:8} ({peakdb})'

        self.header.update_content(
            f'now:{f:8} | '
            f'peak:{peak}'
        )

    def update_xaxis(self) -> None:
        self.xaxis.update_content(make_frequency_labels(
            self.xaxis.width, self.fstart, self.fstop
        ))

    def update_yaxis(self) -> None:
        self.yaxis.update_content(make_db_labels(
            self.yaxis.width,
            self.yaxis.height-1,  # account for xaxis labels
            self.mindbfs,
            self.maxdbfs
        ))

    def update_plot(self) -> None:
        plot = Plot(
            self.plot.width,
            self.plot.height,
            self.fstart,
            self.mindbfs,
            self.fstop,
            self.maxdbfs
        )

        for freq, dbfs in self.signals.items():
            plot.point(freq, dbfs)

        self.plot.update_content(plot.draw())

    def update_sample(self, sample: list[np.complex64]) -> None:
        signals = self.find_signals(sample)
        update = False

        for foff, dbfs in signals:
            freq = self.radio.rx_freq()+(foff-self.radio.rx_bw()//2)

            if freq not in self.signals or self.signals[freq] < dbfs:
                update = True
                self.signals[freq] = dbfs
                if dbfs > self.maxdbfs:
                    self.maxdbfs = round(dbfs)

        if update:
            self.update_yaxis()
            self.update_plot()

    def update_radio(self, radio: Radio) -> None:
        self.radio = radio
        self.update_header()

class PSD(FFTVisualizer):
    def layout(self, container: StyledWindow) -> None:
        dbwidth = max(len(str(self.mindbfs)), len(str(self.maxdbfs)))
        (self.yaxis, body) = container.vsplit(dbwidth)
        (self.plot, self.xaxis) = body.hsplit(None, 1)

        self.windows = [self.plot, self.xaxis, self.yaxis]

        self.plot.update_style(self.style['plot'])
        self.xaxis.update_style(self.style['plot-label'])
        self.yaxis.update_style(self.style['plot-label'])

        self.update_yaxis()
        self.update_xaxis()

    def update_xaxis(self) -> None:
        f = self.radio.rx_freq()
        r = self.radio.rx_bw()
        self.xaxis.update_content(make_frequency_labels(
            self.xaxis.width, f-r/2, f+r/2
        ))

    def update_yaxis(self) -> None:
        self.yaxis.update_content(make_db_labels(
            self.yaxis.width,
            self.yaxis.height-1,  # account for xaxis labels
            self.mindbfs,
            self.maxdbfs
        ))

    def update_plot(self, values: list[float]) -> None:
        plot = Plot(
            self.plot.width,
            self.plot.height,
            0,
            self.mindbfs,
            len(values),
            self.maxdbfs
        )

        (x1, y1) = (0, values[0])
        for x in range(1, len(values)):
            y = values[x]
            plot.line(x1, y1, x, y)
            (x1, y1) = (x, y)

        self.plot.update_content(plot.draw())

    def update_sample(self, sample: list[np.complex64]) -> None:
        self.update_plot(self.get_dbfs(sample))

    def update_radio(self, radio: Radio) -> None:
        self.radio = radio
        self.update_xaxis()

class Waterfall(FFTVisualizer):
    def layout(self, container: StyledWindow) -> None:
        (self.waterfall, self.xaxis) = container.hsplit(None, 1)
        self.windows = [self.waterfall, self.xaxis]

        self.xaxis.update_style(self.style['plot-label'])
        self.update_xaxis()

    def update_xaxis(self) -> None:
        f = self.radio.rx_freq()
        r = self.radio.rx_bw()
        self.xaxis.update_content(make_frequency_labels(
            self.xaxis.width, f-r/2, f+r/2
        ))

    def update_sample(self, sample: list[np.complex64]) -> None:
        def norm(x: float) -> int:
            x = min(max(x, self.mindbfs), self.maxdbfs)
            return int((x-self.mindbfs)/(self.maxdbfs-self.mindbfs) * 255)

        dbfs = self.get_dbfs(sample)

        # lerp to waterfall width and normalize values to 0-255
        values = [norm(x) for x in fit_values(dbfs, self.waterfall.width)]

        # map to rgb value in style colormap
        colors = [self.style['colormap'][x] for x in values]

        # map to styled glyph
        chars = [
            Text(self.style['waterfall-glyph']()).style({
                'bg': self.style['plot']['bg'],
                'fg': x
            })
            for x in colors
        ]

        self.waterfall.append_line(''.join(chars))

    def update_radio(self, radio: Radio) -> None:
        self.radio = radio
        self.update_xaxis()

class Constellation(Visualizer):
    def __init__(
        self,
        radio: Radio,
        iqrange: int = 1,
        style: dict = Styles['tokyonight']
    ) -> None:
        super().__init__(radio, style)
        self.iqrange = iqrange

    def layout(self, container: StyledWindow) -> None:
        self.plot = container
        self.plot.update_style(self.style['plot'])
        self.windows = [self.plot]

    def update_sample(self, sample: list[np.complex64]) -> None:
        r = self.iqrange
        plot = Plot(self.plot.width, self.plot.height, -r, -r, r, r)
        # axis
        plot.line(-r, 0, r, 0)
        plot.line(0, r, 0, -r)
        # iq points
        for point in zip(np.real(sample), np.imag(sample)):
            plot.point(point[0], point[1])
        self.plot.update_content(plot.draw())

    def update_radio(self, radio: Radio) -> None:
        pass

def parse_frange(radio: Radio, options: optparse.Values) -> tuple[int, int]:
    minf = radio.min_freq()
    maxf = radio.max_freq()

    if options.frange is None:
        return (minf, maxf)
    else:
        (fstart, fstop) = options.frange.split(':')
        fstart = minf if fstart == '' else min(maxf, max(minf, int(fstart)))
        fstop = maxf if fstop == '' else min(maxf, max(minf, int(fstop)))
        if fstart > fstop:
            raise optparse.OptionValueError('start frequency higher than stop')
        options.frange = (fstart, fstop)
    return options.frange

def config_visualizer(
    name: str,
    radio: Radio,
    options: optparse.Values
) -> Visualizer:
    (fftsize, nperseg) = (options.fftsize, options.nperseg)
    # default to non-segmented periodogram for RtlRadio
    dnperseg = fftsize if isinstance(radio, RtlRadio) else fftsize//4
    nperseg = dnperseg if nperseg is None else min(fftsize, nperseg)

    if name == 'psd':
        return PSD(
            radio,
            mindbfs=options.mindbfs,
            maxdbfs=options.maxdbfs,
            nperseg=nperseg,
            window=options.window,
            zoff=-1.0,
            style=Styles[options.style]
        )
    elif name == 'waterfall':
        return Waterfall(
            radio,
            mindbfs=options.mindbfs,
            maxdbfs=options.maxdbfs,
            nperseg=nperseg,
            window=options.window,
            zoff=-1.0,
            style=Styles[options.style]
        )
    elif name == 'seek':
        (fstart, fstop) = parse_frange(radio, options)

        return Seek(
            radio,
            fstart=fstart,
            fstop=fstop,
            mindbfs=options.mindbfs,
            nperseg=nperseg,
            window=options.window,
            zoff=-1.0,
            style=Styles[options.style]
        )
    elif name == 'constellation':
        iqrange = 16384 if isinstance(radio, PlutoRadio) else 1
        return Constellation(radio, iqrange, Styles[options.style])
    else:
        raise optparse.OptionValueError(f'unknown visualizer {name}.')


import argparse

from collections import namedtuple
from datetime import datetime

# from typing import NamedTuple

DATETIME_SUPPORTED_FORMATS = ["%d-%m-%Y", "%Y-%m-%d"]

# We could do sth like this to add typing to our src/dst/date attributes
# class Request(NamedTuple):
#     src: str
#     dst: str
#     date: datetime

class Request(namedtuple("request", "src dst date")):
    @staticmethod
    def date_type(date_string):
        for fmt in DATETIME_SUPPORTED_FORMATS:
            try:
                return datetime.strptime(date_string, fmt)
            except ValueError:
                pass

        msg = f"date {date_string} doesn't match formats {DATETIME_SUPPORTED_FORMATS}"
        raise NotImplementedError(msg)

    @classmethod
    def from_cli(cls):
        parser = argparse.ArgumentParser()

        parser.add_argument("src", nargs="?")
        parser.add_argument("dst", nargs="?")
        parser.add_argument("date", nargs="?", type=cls.date_type)

        args = parser.parse_args()

        return cls(args.src, args.dst, args.date)


import os
import cv2
import numpy as np
import matplotlib.pyplot as plt

import random as rng

def pltShowCV(img, title="img"):
    if len(img.shape) == 2:
        img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)
        print('Image is single-channel (grayscale)')
    elif len(img.shape) == 3:
        print('Image is three-channel (RGB)')
        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    plt.imshow(img)
    plt.title(title)
    plt.show()

def pltShowCVDot(img, point, title="img"):
    global cnt
    if len(img.shape) == 2:
        img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)
        print('Image is single-channel (grayscale)')
    elif len(img.shape) == 3:
        print('Image is three-channel (RGB)')
        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    cv2.circle(img, point, 20, (255, 0, 0), -1)
    plt.imshow(img)
    plt.title(title + str(cnt))
    plt.show()
    cnt += 1

def draw_contours(img_gray, contours, hierarchy):
    drawing = np.zeros((img_gray.shape[0], img_gray.shape[1], 3), dtype=np.uint8)
    for i in range(len(contours)):
        color = (rng.randint(0,256), rng.randint(0,256), rng.randint(0,256))
        cv2.drawContours(drawing, contours, i, color, 2, cv2.LINE_8, hierarchy, 0)
    return drawing

class _TKDWater_Defect:
    def __init__(self):
        self.rtRect = None
        self.iArea = 0

class CAlgo_KDWater:
    def __init__(self):
        self.m_imgTempl = None
        self.m_ptStdOffs = None
        self.DEF_THRESD_AREA = 0
        self.DEF_THRESD_NOR = 50
        self.DEF_THRESD_DIFF = 60

    def LoadTemplate(self):
        self.m_imgTempl = cv2.imread("Template\\nm_block.bmp", cv2.IMREAD_GRAYSCALE)
        self.m_ptStdOffs = (213, 213)
        return True

    def FindDefect(self, vecDefect, srcImgGray):
        vecDefect.clear()

        ptCent = [0, 0]
        if self._GetBestMatchPos(ptCent, srcImgGray):
            self._SearchDefect(vecDefect, ptCent, srcImgGray)

    def _SearchDefect(self, vecDefect, cent, srcImgGray):
        global filename
        vecBoxCent = [cent]
        iIndex_R = 0

        ret = True
        while iIndex_R < len(vecBoxCent):
            curCent = vecBoxCent[iIndex_R]
            iIndex_R += 1
            pltShowCVDot(srcImgGray, curCent, "curCent")
            self._SearchDefect_Box(vecDefect, vecBoxCent, curCent, srcImgGray)
            if len(vecDefect) > 0:
                # ret = False
                # break
                pass

        imgColor = cv2.cvtColor(srcImgGray, cv2.COLOR_GRAY2BGR)

        for idx in range(len(vecDefect)):
            boxRT = vecDefect[idx].rtRect
            boxRT[0] -= 5
            boxRT[1] -= 5
            boxRT[2] += 10
            boxRT[3] += 10
            if boxRT[0] < 0:
                boxRT[0] = 0
            if boxRT[1] < 0:
                boxRT[1] = 0
            if boxRT[0] + boxRT[2] > imgColor.shape[1]:
                boxRT[2] = imgColor.shape[1] - boxRT[0]
            if boxRT[1] + boxRT[3] > imgColor.shape[0]:
                boxRT[3] = imgColor.shape[0] - boxRT[1]

            cv2.rectangle(imgColor, boxRT, (0, 0, 255), 2)
            # buf = f"{vecDefect[idx].iArea}"
            # cv2.putText(imgColor, buf, vecDefect[idx].rtRect.tl(), cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 0, 255), 2)

        imgOut = imgColor.copy()

        iHalfW = self.m_imgTempl.shape[1] >> 1
        iHalfH = self.m_imgTempl.shape[0] >> 1
        for idx in range(len(vecBoxCent)):
            rect = (vecBoxCent[idx][0] - iHalfW, vecBoxCent[idx][1] - iHalfH, self.m_imgTempl.shape[1],
                    self.m_imgTempl.shape[0])
            cv2.rectangle(imgOut, rect, (0, 255, 0), 2)

        buf = [0] * 128
        cnt = 0
        if len(vecDefect) > 0:
            cnt += 1
            buf = f"d:\\1\\{filename}_[{len(vecDefect)}]_NG.bmp"
            cv2.imwrite(buf, imgColor)
        else:
            cnt += 1
            buf = f"d:\\1\\{filename}_OK.bmp"
            cv2.imwrite(buf, imgColor)

        # imgC2 = cv2.pyrDown(imgColor)
        # cv2.imshow("1", imgC2)
        # cv2.waitKey(0)

        if len(vecDefect) > 0:
            ret = False

        return ret

    def _addBoxPos(self, vecBoxCent, cent):
        total = len(vecBoxCent)
        ret = True
        for idx in range(total):
            dx = abs(vecBoxCent[idx][0] - cent[0])
            dy = abs(vecBoxCent[idx][1] - cent[1])

            if dx < 100 and dy < 100:
                ret = False
                break

        if ret:
            vecBoxCent.append(cent)

    #
    def _SearchDefect_Box(self, vecDefect, vecBoxCent, cent, srcImgGray):
        curRect = (
            cent[0] - (self.m_imgTempl.shape[1] >> 1), cent[1] - (self.m_imgTempl.shape[0] >> 1),
            self.m_imgTempl.shape[1],
            self.m_imgTempl.shape[0])
        imgCur = srcImgGray[curRect[1]:curRect[1] + curRect[3], curRect[0]:curRect[0] + curRect[2]]

        bNeib = [False] * 4
        imgNeib = [np.zeros((251, 247), dtype=np.uint8) for i in range(4)]
        ptNeib = [[0, 0] for i in range(4)]

        funcList = [self._GetImage_ByBox_Left, self._GetImage_ByBox_Right, self._GetImage_ByBox_Up, self._GetImage_ByBox_Down]
        for idx in range(4):
            ret, imgNeib[idx] = funcList[idx](imgNeib[idx], ptNeib[idx], srcImgGray, cent, imgCur)
            if ret:
                bNeib[idx] = True
                self._addBoxPos(vecBoxCent, ptNeib[idx])
            else:
                imgNeib[idx] = imgCur.copy()

        DEF_THRESD_AREA = self.DEF_THRESD_AREA
        DEF_THRESD_NOR = self.DEF_THRESD_NOR        # 50
        DEF_THRESD_DIFF = self.DEF_THRESD_DIFF      # 60

        diff = [np.zeros((1, 1), dtype=np.uint8)] * 4
        for idx in range(4):
            diff[idx] = cv2.absdiff(imgNeib[idx], imgCur)
            _, diff[idx] = cv2.threshold(diff[idx], DEF_THRESD_DIFF, 1, cv2.THRESH_BINARY)

        diff[0] += diff[1] + diff[2] + diff[3]

        imgBin = cv2.threshold(diff[0], 1, 255, cv2.THRESH_BINARY)[1]

        imgBaseBin = cv2.threshold(imgCur, DEF_THRESD_NOR, 255, cv2.THRESH_BINARY_INV)[1]
        imgBin &= imgBaseBin

        kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (7, 7))
        imgBin = cv2.erode(imgBin, kernel)
        imgBin = cv2.dilate(imgBin, kernel)

        kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (7, 7))
        imgBin = cv2.dilate(imgBin, kernel)
        imgBin = cv2.dilate(imgBin, kernel)
        imgBin = cv2.erode(imgBin, kernel)
        imgBin = cv2.erode(imgBin, kernel)
        pltShowCV(imgBin, "imgBin")
        contours, hierarchy = cv2.findContours(imgBin, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
        drawing = draw_contours(imgBin, contours, hierarchy)
        pltShowCV(drawing, "drawing")
        #vecDefect = []
        for cnt in contours:
            area = cv2.contourArea(cnt)
            if area > DEF_THRESD_AREA:
                defect = _TKDWater_Defect()
                defect.iArea = area
                defect.rtRect = list(cv2.boundingRect(cnt))
                defect.rtRect[0] += curRect[0]
                defect.rtRect[1] += curRect[1]
                vecDefect.append(defect)

    def _GetImage_ByBox_Left(self, outImg, outCent, srcImg, cent, curImgTempl):
        iHalfWid = (self.m_imgTempl.shape[1] >> 1) + 30
        iHalfHei = (self.m_imgTempl.shape[0] >> 1) + 30

        roiRect = (cent[0] - iHalfWid - self.m_ptStdOffs[0], cent[1] - iHalfHei, iHalfWid << 1, iHalfHei << 1)
        if roiRect[0] < 0 or roiRect[1] < 0 or roiRect[0] + roiRect[2] > srcImg.shape[1] or roiRect[1] + roiRect[3] > \
                srcImg.shape[0]:
            return False, None

        roiImg = srcImg[roiRect[1]:roiRect[1] + roiRect[3], roiRect[0]:roiRect[0] + roiRect[2]]
        ret, outImg = self._GetMaxMatch(outImg, outCent, roiImg, curImgTempl)
        if ret:
            outCent[0] += roiRect[0]
            outCent[1] += roiRect[1]

        return ret, outImg

    def _GetImage_ByBox_Right(self, outImg, outCent, srcImg, cent, curImgTempl):
        iHalfWid = (self.m_imgTempl.shape[1] >> 1) + 30
        iHalfHei = (self.m_imgTempl.shape[0] >> 1) + 30

        roiRect = (cent[0] - iHalfWid + self.m_ptStdOffs[0], cent[1] - iHalfHei, iHalfWid << 1, iHalfHei << 1)
        if (roiRect[0] < 0) or (roiRect[1] < 0) or (roiRect[0] + roiRect[2] > srcImg.shape[1]) or (
                roiRect[1] + roiRect[3] > srcImg.shape[0]):
            return False, None

        roiImg = srcImg[roiRect[1]:roiRect[1] + roiRect[3], roiRect[0]:roiRect[0] + roiRect[2]]
        ret, outImg = self._GetMaxMatch(outImg, outCent, roiImg, curImgTempl)
        if ret:
            outCent[0] += roiRect[0]
            outCent[1] += roiRect[1]

        return ret, outImg

    def _GetImage_ByBox_Up(self, outImg, outCent, srcImg, cent, curImgTempl):
        iHalfWid = (self.m_imgTempl.shape[1] >> 1) + 30
        iHalfHei = (self.m_imgTempl.shape[0] >> 1) + 30

        roiRect = (cent[0] - iHalfWid, cent[1] - iHalfHei - self.m_ptStdOffs[1], iHalfWid << 1, iHalfHei << 1)
        if (roiRect[0] < 0) or (roiRect[1] < 0) or (roiRect[0] + roiRect[2] > srcImg.shape[1]) or (
                roiRect[1] + roiRect[3] > srcImg.shape[0]):
            return False, None

        roiImg = srcImg[roiRect[1]:roiRect[1] + roiRect[3], roiRect[0]:roiRect[0] + roiRect[2]]
        ret, outImg = self._GetMaxMatch(outImg, outCent, roiImg, curImgTempl)
        if ret:
            outCent[0] += roiRect[0]
            outCent[1] += roiRect[1]

        return ret, outImg

    def _GetImage_ByBox_Down(self, outImg, outCent, srcImg, cent, curImgTempl):
        iHalfWid = (self.m_imgTempl.shape[1] >> 1) + 30
        iHalfHei = (self.m_imgTempl.shape[0] >> 1) + 30

        roiRect = (cent[0] - iHalfWid, cent[1] - iHalfHei + self.m_ptStdOffs[1], iHalfWid << 1, iHalfHei << 1)
        if roiRect[0] < 0 or roiRect[1] < 0 or roiRect[0] + roiRect[2] > srcImg.shape[1] or roiRect[1] + roiRect[3] > \
                srcImg.shape[0]:
            return False, None

        roiImg = srcImg[roiRect[1]:roiRect[1] + roiRect[3], roiRect[0]:roiRect[0] + roiRect[2]]
        ret, outImg = self._GetMaxMatch(outImg, outCent, roiImg, curImgTempl)
        if ret:
            outCent[0] += roiRect[0]
            outCent[1] += roiRect[1]

        return ret, outImg

    def _GetMaxMatch(self, outImg, outCent, roiImg, imgTempl):
        result = cv2.matchTemplate(roiImg, imgTempl, cv2.TM_CCOEFF_NORMED)
        result = np.clip(result, 0, np.inf)
        result = result * 255
        #result = cv2.normalize(result, None, 0, 255, cv2.NORM_MINMAX, cv2.CV_8U)

        dMax = 0
        _, dMax, _, maxLoc = cv2.minMaxLoc(result)
        iVal = dMax
        if iVal < 100:
            return False, None
        iSumX = 0
        iSumY = 0
        cnt = 0
        for row in range(result.shape[0]):
            pLine = result[row, :]
            for col in range(result.shape[1]):
                if pLine[col] >= iVal:
                    iSumX += col
                    iSumY += row
                    cnt += 1

        if cnt <= 0:
            return False

        outCent[0] = iSumX // cnt
        outCent[1] = iSumY // cnt

        roiRect = (outCent[0], outCent[1], imgTempl.shape[1], imgTempl.shape[0])    # row:行数 高度 251  cols: 宽度 247
        outImg = roiImg[roiRect[1]:roiRect[1] + roiRect[3], roiRect[0]:roiRect[0] + roiRect[2]]

        outCent[0] += imgTempl.shape[1] // 2
        outCent[1] += imgTempl.shape[0] // 2

        return True, outImg

    def _GetMaxMatch_jianhua(self, outImg, outCent, roiImg, imgTempl):
        result = cv2.matchTemplate(roiImg, imgTempl, cv2.TM_CCOEFF_NORMED)
        result = cv2.normalize(result, None, 0, 255, cv2.NORM_MINMAX, cv2.CV_8U)

        dMax = 0
        _, dMax, _, maxLoc = cv2.minMaxLoc(result)
        if dMax < 100:
            return False

        outCent[0] = maxLoc[0] + imgTempl.shape[1] // 2
        outCent[1] = maxLoc[1] + imgTempl.shape[0] // 2

        roiRect = (maxLoc[0], maxLoc[1], imgTempl.shape[1], imgTempl.shape[0])
        outImg = roiImg[roiRect[1]:roiRect[1] + roiRect[3], roiRect[0]:roiRect[0] + roiRect[2]]

        return True

    def _GetBestMatchPos(self, ptCent, srcImgGray):
        result = cv2.matchTemplate(srcImgGray, self.m_imgTempl, cv2.TM_CCOEFF_NORMED)
        # result = result * 255
        # result = np.uint8(result)
        result = cv2.normalize(result, None, 0, 255, cv2.NORM_MINMAX, cv2.CV_8U)

        dMax = 0
        _, dMax, _, _ = cv2.minMaxLoc(result)

        dMax -= 1.0
        if dMax < 100:
            dMax = 100

        _, imgBin = cv2.threshold(result, dMax, 255, cv2.THRESH_BINARY)

        vecCont, _ = cv2.findContours(imgBin, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
        vecRect = []
        iMaxV = 0
        iMaxIdx = -1
        for idx in range(len(vecCont)):
            area = len(vecCont[idx])
            if area > iMaxV:
                iMaxV = area
                iMaxIdx = idx

        ret = False

        if iMaxIdx >= 0:
            rect = cv2.boundingRect(vecCont[iMaxIdx])

            ptCent[0] = rect[0] + (rect[2] >> 1) + (self.m_imgTempl.shape[1] >> 1)
            ptCent[1] = rect[1] + (rect[3] >> 1) + (self.m_imgTempl.shape[0] >> 1)

            ret = True

        return ret

        # DEF_THRESD_AREA = self.DEF_THRESD_AREA
        # DEF_THRESD_NOR = self.DEF_THRESD_NOR        # 50
        # DEF_THRESD_DIFF = self.DEF_THRESD_DIFF      # 60
    def setDEF_THRESD_AREA(self, DEF_THRESD_AREA):
        self.DEF_THRESD_AREA = DEF_THRESD_AREA

    def setDEF_THRESD_NOR(self, DEF_THRESD_NOR):
        self.DEF_THRESD_NOR = DEF_THRESD_NOR

    def setDEF_THRESD_DIFF(self, DEF_THRESD_DIFF):
        self.DEF_THRESD_DIFF = DEF_THRESD_DIFF

def test_water():
    global filename
    water = CAlgo_KDWater()
    water.LoadTemplate()

    vecFiles = []
    _scan_files("D:\\LGCBTray\\Test_Image\\NM", vecFiles)

    vecDefect = []
    for idx in range(len(vecFiles)):
        img = cv2.imread(vecFiles[idx], cv2.IMREAD_GRAYSCALE)
        pltShowCV(img, "OriImg")
        file_name = os.path.basename(vecFiles[idx])
        filename = os.path.splitext(file_name)[0]
        water.FindDefect(vecDefect, img)

def _scan_files(directory, vecFiles):
    for file in os.listdir(directory):
        if file.endswith(".jpg") or file.endswith(".png") or file.endswith(".bmp"):
            vecFiles.append(os.path.join(directory, file))

filename = ""
cnt = 1
if __name__ == '__main__':
    test_water()

"""
    cv2.imwrite("test.jpg", result)
"""


from flask import request, jsonify
from models import (
    Movie,
    Actor,
    MovieActor
)

def create_movie():
    data_json = request.get_json()
    title = data_json.get('title')
    genre = data_json.get('genre')
    if title is None:
        return 'valid title is required', 400
    if genre is None:
        return 'valid genre is required', 400
    m = Movie.get_or_create(
        title=title,
        genre=genre
    )
    actor_names = data_json.get('actors')
    if actor_names is not None:
        for actor_name in actor_names:
            a = Actor.get_or_create(actor_name)
            MovieActor.get_or_create(
                actor_uuid=a.uuid,
                movie_uuid=m.uuid
            )
    return jsonify(m.serialize_to_dict()), 200

def get_movies():
    genre_filter = request.args.get('genre')
    actor_filter = request.args.get('actor')

    movies = Movie.select()
    # performance improvement can be made by having slightly uglier code
    if actor_filter is not None:
        movies = movies.join(MovieActor).join(Actor).where(
            Actor.name == actor_filter
        )
    if genre_filter is not None:
        movies = movies.where(
            Movie.genre == genre_filter
        )
    return jsonify([movie.serialize_to_dict() for movie in movies])


import pygame, os, turtle, sys, time

pygame.init() #Iniciamos pygame
tiempo = time.time()

def collide_DOWN(jugador): #Funcion para detectar colisiones con los bordes inferiores
    choque = False
    for i in range (1, largo_jugador):
        pixeles_fondo = pygame.Surface.get_at(fondo, (jugador.x + i, jugador.y + 20))
        if pixeles_fondo == (0, 0, 0, 255):
            choque = True
    return choque

def collide_UP(jugador): #Funcion para detectar colisiones con los bordes superiores
    choque = False
    for i in range (1, largo_jugador):
        pixeles_fondo = pygame.Surface.get_at(fondo, (jugador.x + i, jugador.y))
        if pixeles_fondo == (0, 0, 0, 255):
            choque = True
    return choque

def collide_RIGHT(jugador): #Funcion para detectar colisiones con los bordes derechos
    choque = False
    for i in range (1, largo_jugador):
        pixeles_fondo = pygame.Surface.get_at(fondo, (jugador.x + 20, jugador.y + i))
        if pixeles_fondo == (0, 0, 0, 255):
            choque = True
    return choque

def collide_LEFT(jugador): #Funcion para detectar colisiones con los bordes izquierdos
    choque = False
    for i in range (1, largo_jugador):
        pixeles_fondo = pygame.Surface.get_at(fondo, (jugador.x, jugador.y + i))
        if pixeles_fondo == (0, 0, 0, 255):
            choque = True
    return choque

def WIN_GAME(jugador): #Funcion para el juego ganado
    global tiempo_final
    if jugador.y > alto - alto_jugador - 4:
        run = False
        print(tiempo_final)
        WIN_IMAGE()

def LOSE_GAME(jugador): #Funcion para el juego perdido
    global tiempo_final
    run = False
    print(tiempo_final)
    LOSE_IMAGE()

def tiempo_transcurrido(): #Funcion para el tiempo transcurrido
    global tiempo
    tiempo_actual = time.time() #Tiempo actual
    tiempo_transcurrido = tiempo_actual - tiempo #Calculamos el tiempo transcurrido
    return tiempo_transcurrido

def movimiento_jugador(keys_pressed, jugador): #Funcion para el movimiento del jugador
    global CONT_PASOS
    if keys_pressed[pygame.K_a] or keys_pressed[pygame.K_LEFT]: #SI SE APRETA A O FLECHA IZQUIERDA SE MUEVE 1 PIXELES HACÍA LA IZQUIERDA
        if collide_LEFT(jugador) == False:
            jugador.x -= vel
            CONT_PASOS += 1
    if keys_pressed[pygame.K_d] or keys_pressed[pygame.K_RIGHT]: #SI SE APRETA D O FLECHA DERECHA SE MUEVE 1 PIXELES HACÍA LA DERECHA
        if collide_RIGHT(jugador) == False:
            jugador.x += vel
            CONT_PASOS += 1
    if keys_pressed[pygame.K_w] or keys_pressed[pygame.K_UP]: #SI SE APRETA W O FLECHA SUPERIOR SE MUEVE 1 PIXELES HACÍA ARRIBA
        if collide_UP(jugador) == False:
            jugador.y -= vel
            CONT_PASOS += 1
    if keys_pressed[pygame.K_s] or keys_pressed[pygame.K_DOWN]: #SI SE APRETA S O FLECHA INFERIOR SE MUEVE 1 PIXELES HACÍA ABAJO
        if collide_DOWN(jugador) == False:
            jugador.y += vel
            CONT_PASOS += 1

def WIN_IMAGE(): #Funcion para la imagen de ganado
    pygame.init()
    ven = pygame.display.set_mode((800,600))
    pygame.display.set_caption("Juego terminado")
    Fuente = pygame.font.SysFont("Arial", 80)
    while True:
        ven.fill((blanco))  #color de fondo
        for event in pygame.event.get():
            if event.type==pygame.QUIT:
                pygame.quit()
                sys.exit()
            texto=Fuente.render("¡YOU WIN!",0,(azul))
            ven.blit(texto,(230,230))
            pygame.display.update()

def LOSE_IMAGE(): #Funcion para la imagen de perdida
    pygame.init()
    ven = pygame.display.set_mode((800,600))
    pygame.display.set_caption("Juego terminado")
    Fuente = pygame.font.SysFont("Arial", 80)
    while True:
        ven.fill((blanco))
        for event in pygame.event.get():
            if event.type==pygame.QUIT:
                pygame.quit()
                sys.exit()
            texto=Fuente.render("¡YOU LOSE!",0,(rojo))
            ven.blit(texto,(230,230))
            pygame.display.update()

def mov_imagenes(jugador): #Funcion para las imagenes en el juego
    global tiempo_final
    ventana.fill(blanco)
    ventana.blit(fondo, (0, 0))
    ventana.blit(imagen_jugador, (jugador.x, jugador.y))
    PASOS = pasos_fuente.render("Pasos: " + str(CONT_PASOS), 1, rojo)
    TIEMPO = tiempo_fuente.render("Tiempo: " + str((int(tiempo_final))), 1, rojo)
    ventana.blit(PASOS, (ancho - PASOS.get_width() - 10, 590))
    ventana.blit(TIEMPO, (10, 590))
    pygame.display.update()

def main(): #Funcion principal del juego
    global tiempo_final
    jugador = pygame.Rect(ancho/2, alto/2, largo_jugador, alto_jugador) #Creacion del jugador
    clock = pygame.time.Clock()
    run = True
    #PROGRAMA MAIN
    while run == True: #Ciclo principal del juego

        clock.tick(FPS) #Se limitan los fotogramas a "FPS"
        for event in pygame.event.get():
            if event.type == pygame.QUIT: #Se terminara el bucle si se cierra la ventana.

                run = False
        keys_pressed = pygame.key.get_pressed() #Se obtienen las teclas presionadas
        tiempo_final = tiempo_transcurrido()
        movimiento_jugador(keys_pressed, jugador) #Se llama a la funcion para el movimiento del jugador
        mov_imagenes(jugador) #Se llama a la funcion para las imagenes en el juego
        WIN_GAME(jugador) #Se llama a la funcion para ver si gana
        if tiempo_final > tiempo_limite:
            LOSE_GAME(jugador) #Se llama a la funcion de que perdió

#COLORES
blanco = (255, 255, 255)
negro = (0, 0, 0)
rojo = (255, 0, 0)
azul = (0, 0, 255)

#CONSTANTES
vel = 1 #Velocidad del jugador
FPS = 60 #Fotogramas por segundo que tendra el programa
ancho, alto = 800, 600 #Dimensiones de la ventana del juego
tiempo_limite = 50 #Tiempo limite para ganar

#CONTADOR
CONT_PASOS = 0 #Contador de pasos

#LETRA PASOS
pasos_fuente = pygame.font.SysFont('comicsans', 40) #Fuente para los pasos
tiempo_fuente = pygame.font.SysFont('comicsans', 40) #Fuente para el tiempo

#PANTALLA
ventana = pygame.display.set_mode((ancho, alto+50)) #Creacion de la ventana del juego
pygame.display.set_caption("Laberinto del fauno") #NOMBRE DEL JUEGO

fondo = pygame.image.load("laberinto.svg") #Cargamos la imagen que representara al jugador
fondo = pygame.transform.rotate(pygame.transform.scale(fondo, (ancho, alto)), 180) # Rotamos y escalamos la imagen

largo_jugador, alto_jugador = 20, 20 #Dimensiones del jugador
imagen_jugador = pygame.transform.scale(pygame.image.load("circ_rojo.png"), (largo_jugador, alto_jugador)) #Cargamos la imagen del jugador y la escalamos

main()


import numpy as np
import matplotlib.pyplot as plt

lr_dir = './'

f = open(f'{lr_dir}plot_data.txt', 'r')
epochs_eval = []
eval_loss = []
bleu1 = []
bleu2 = []
bs_p = []
bs_r = []
bs_f1 = []
line = f.readline()
while line != '':
    tmp = line.split(',')
    eval_loss.append(float(tmp[0].split(':')[1]))
    bleu1.append(float(tmp[1].split(':')[1]))
    bleu2.append(float(tmp[2].split(':')[1]))
    bs_p.append(float(tmp[3].split(':')[1]))
    bs_r.append(float(tmp[4].split(':')[1]))
    bs_f1.append(float(tmp[5].split(':')[1]))
    epochs_eval.append(float(tmp[-1].split(':')[1].replace('}', '')))
    line = f.readline()

f = open(f'{lr_dir}plot_train_loss_data.txt', 'r')
epochs_train = []
train_loss = []
line = f.readline()
while line != '':
    train_loss.append(float(line.split(',')[0].split(':')[1]))
    epochs_train.append(float(line.replace('}', '').split(':')[-1]))
    line = f.readline()

# Plot BLEU seperately for good scale
plt.plot(epochs_eval, eval_loss, label='Eval Loss')
plt.plot(epochs_train, train_loss, label='Train Loss')

plt.title('Loss Over Time', y=1)
plt.ylabel('Loss')
plt.xlabel('Epochs Trained')
plt.legend()
plt.savefig(f'{lr_dir}eval_loss.png')
plt.clf()

plt.plot(epochs_eval, bs_p, label='BERTScore Precision')
plt.plot(epochs_eval, bs_r, label='BERTScore Recall')
plt.plot(epochs_eval, bs_f1, label='BERTScore F1')

plt.title('BERTScores Over Time', y=1)
plt.ylabel('Score')
plt.xlabel('Epochs Trained')
plt.legend()
plt.savefig(f'{lr_dir}bertscores.png')
plt.clf()

# Plot BLEU seperately for good scale
plt.plot(epochs_eval, bleu1, label='Bleu-1 Score')
plt.plot(epochs_eval, bleu2, label='Bleu-2 Score')

plt.title('Bleu Scores Over Time', y=1)
plt.ylabel('Score')
plt.xlabel('Epochs Trained')
plt.legend()
plt.savefig(f'{lr_dir}bleuscore.png')


import openai

openai.api_key = ''
with open("input.txt", "rt") as file:
    response = openai.Completion.create(
      model="text-davinci-003",
      prompt=file.read(),
      temperature=0.7,
      max_tokens=256,
      top_p=1,
      frequency_penalty=0,
      presence_penalty=0
    )

print("\noutput.py\n", response["choices"][0]["text"])
with open("output.py", "wt") as file:
    file.write(response["choices"][0]["text"])


# Importing dependencies
import pandas as pd
import datetime
from pathlib import Path

class DF_Processor:
    """
    This class reads, processes and returns both regional and non-regional CSVs as DataFrames from the Data directory.
    The __init__ (constructor) method sets both the regional and non-regional CSV titles as class variables.
    The get_dfs method takes in the is_regional boolean flag and outputs a list of all of the requested DataFrames.
    The process_dfs method takes care of all the necessary DataFrame processing we need to get our data in the right form.
    The get_regional_data method returns a list of the regional DataFrames.
    And the get_non_regional_data returns a list of the non-regional DataFrames.
    """

    # Constructor
    def __init__(self):
        # List of non-regional csv titles
        self.non_regional_titles = ['National','Energy','Food', 'National-Less-Food-Energy']
        # List of regional csv titles
        self.regional_titles = ['Midwest', 'Northeast', 'South', 'West']

    # Method to get dataframes
    def get_dfs(self, is_regional):
        # Empty list to hold DataFrames
        dataframes = []

        # Setting titles list
        titles = self.regional_titles if is_regional else self.non_regional_titles

        # Looping over csv titles list
        for title in titles:
            # Columns to drop
            columns_to_drop = ['HALF1','HALF2']

            # If the regional flag is set
            if is_regional:
                # Append the Annual column
                columns_to_drop.append('Annual')

            # Saving the csv (by title) as DataFrame, dropping un-needed comments, and reshaping using melt
            df = pd.read_csv(Path(f'./Data/{title}.csv')).drop(columns=columns_to_drop).melt(id_vars=['Year'], var_name='Month', value_name='Price')

            # Appending dataframe to dataframes list
            dataframes.append(df)

        # Returning all dataframes
        return dataframes

    # Method to data pre-process dataframes
    def process_dfs(self, is_regional):
        # Empty list to hold processed DataFrames
        processed_dataframes = []
        # Dictionary of months and thier numeric values
        months = {'Jan':1,'Feb':2,'Mar':3,'Apr':4,'May':5,'Jun':6,'Jul':7,'Aug':8,'Sep':9,'Oct':10,'Nov':11,'Dec':12}
        # Get and loop over DataFrames
        for df in self.get_dfs(is_regional):
            # Creating Day column and setting values to 1
            df['Day']=1
            # Mapping month from string to int
            df['Month'] = df['Month'].map(months)

            # Creating date column
            df['Date']=pd.to_datetime(dict(year=df.Year, month=df.Month, day=df.Day))
            # Sorting Date values
            df = df.sort_values('Date')
            # Making index Date
            df.index = df['Date']

            # Dropping Year, Month, Day, Date columns
            df = df.drop(columns=['Year','Month','Day','Date'])
            # Dropping NaN values
            df = df.dropna()

            # Adding current DataFrame to processed_dataframes list
            processed_dataframes.append(df)
        #Returning all of the processed dataframes
        return processed_dataframes

    # Method to get regional DataFrames
    def get_regional_data(self):
        # Return the processed regional DataFrames
        return self.process_dfs(True)

    # Method to get non-regional DataFrames
    def get_non_regional_data(self):
        # Return the processed non-regional DataFrames
        return self.process_dfs(False)


import pika
import numpy as np
from src.lib.generators import Meter, Photovoltaic
from src.lib.metermessage import MeterMessage

import threading
import time

class ThreadTimer(object):
    # Timer for simulation. Uses a Mutex as it should be shared if simulating multiple households
    def __init__(self, init_time=0, duration=3600):
        self.mutex = threading.Lock()
        self.duration = duration
        self.init_time = init_time
        self.time = init_time

    def end(self):
        with self.mutex:
            return self.duration + self.init_time <= self.time

    def increment_time(self, interval=1):
        with self.mutex:
            self.time += interval

    def get_time(self):
        self.mutex
        with self.mutex:
            return self.time

class BrokerConfigs(object):
    # Configurations for RabbitMQ Channel
    def __init__(self, queue='default', host='localhost', routing_key='default', exchange='', credentials=pika.PlainCredentials('guest', 'guest')):
        self.credentials = credentials
        self.queue = str(queue)
        self.host = str(host)
        self.routing_key = str(routing_key)
        self.exchange = str(exchange)

class ThreadHousehold(object):
    # Thread simulating Household, Starts when start_event is set, stops when stop_event is set
    def __init__(self, start_event, stop_event, timer: ThreadTimer=None, meter=Meter(), brokerconfigs=BrokerConfigs()):
        self.start_event = start_event
        self.stop_event = stop_event
        self.meter = meter

        self.timer = timer

        self.config = brokerconfigs
        self.connection = pika.BlockingConnection(pika.ConnectionParameters(host=self.config.host, credentials=self.config.credentials))
        self.channel = self.connection.channel()
        self.channel.queue_declare(queue=self.config.queue)

        thread = threading.Thread(target=self._run)
        thread.start()

    def _publish(self, message=''):
        self.channel.basic_publish(exchange=self.config.exchange, routing_key=self.config.routing_key, body=message)

    def _run(self):
        while not self.start_event.is_set():
            time.sleep(0.01)

        print('Household thread Started')

        t_last = 0

        while not self.stop_event.is_set():
            if self.timer is not None:
                t = self.timer.get_time()
            else:
                t = int(time.time())
                # Only send message if 2 seconds has passed
            if t_last + 2 <= t:
                m = MeterMessage(origin=self.meter.id, time=t, consumption=self.meter.get_consumption(t))
                self._publish(message=str(m.to_json()))
                t_last = t
            if self.timer is not None:
                self.timer.increment_time(2)

            time.sleep(0.01)

        print('Household thread Finished')

        self.connection.close()

class ThreadPvGenerator(object):
    # Thread simulating Generator, Starts when start_event is set, stops when stop_event is set
    def __init__(self, start_event, stop_event, photovoltaic=Photovoltaic(), brokerconfigs=BrokerConfigs(), logfile=None):
        self.start_event = start_event
        self.stop_event = stop_event

        if logfile is not None:
            self.logfile = open(logfile, 'w')

            self.logfile.write(f"{'Time':>12} [s] | {'Meter Name':^10}{'Consumption':^17} [W] |{'PV Name':^10}{'Generation':^20} [kW] |{'Netto':^20} [kW]\n")
        else:
            self.logfile = None

        self.photovoltaic = photovoltaic

        self.config = brokerconfigs
        self.connection = pika.BlockingConnection(pika.ConnectionParameters(host=self.config.host, credentials=self.config.credentials))
        self.channel = self.connection.channel()
        self.channel.queue_declare(queue=self.config.queue)

        thread = threading.Thread(target=self._run)
        thread.setDaemon(True)
        thread.start()

    def _run(self):
        while not self.start_event.is_set():
            time.sleep(0.01)

        print("PV Thread Start")

        self.channel.basic_consume(queue=self.config.queue, on_message_callback=self._on_request, auto_ack=True)
        while not self.stop_event.is_set():
            self.connection.process_data_events()
            time.sleep(0.1)
        self.connection.close()
        print("PV thread finished")

    def _on_request(self, ch, method, properties, body):
        # self.queue[properties.correlation_id] = body
        m = MeterMessage.from_json(body)
        self.logger(message=m)

    def logger(self, message: MeterMessage = ''):
        generation = self.photovoltaic.get_power(message.time)
        str = "{:>12} [s] | {:^10} consumed {:<07} [W] | {:^10} generated {:<08} [kW] | Netto: {:<012} [kW]\n"\
            .format(message.time, message.origin, np.round(message.consumption, 2), self.photovoltaic.id,\
                np.round(generation/1000.0, 6), np.round((generation - message.consumption)/1000, 6))
        if self.logfile is not None:
            self.logfile.write(str)
            return
        print(str)

    def __del__(self):
        self.stop_event.set()
        print("closing thread")
        if self.logfile is not None:
            self.logfile.close()

if __name__ == '__main__':
    exit(0)


"""initial migration

Revision ID: f52c4f9eb3d5
Revises: c83b808fab81
Create Date: 2017-05-22 13:55:52.981000

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'f52c4f9eb3d5'
down_revision = 'c83b808fab81'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('users', sa.Column('create_time', sa.DateTime(), nullable=True))
    op.add_column('users', sa.Column('group_id', sa.Integer(), nullable=True))
    op.add_column('users', sa.Column('update_time', sa.DateTime(), nullable=True))
    op.create_foreign_key(None, 'users', 'users_group', ['group_id'], ['id'])
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_constraint(None, 'users', type_='foreignkey')
    op.drop_column('users', 'update_time')
    op.drop_column('users', 'group_id')
    op.drop_column('users', 'create_time')
    # ### end Alembic commands ###


import numpy as np
import matplotlib.pyplot as plt
from pathlib import Path
from os import sep

home = str(Path.home())
cur_dir = str(Path.cwd())
layer_number = [2, 4, 8]
neuron_number = [20, 10, 5]
training_number = 100
value_number = 10000
val_range = [np.pi/2-.17, -(np.pi/2-.17)]

#### Print out Variables ####
x = np.load(''.join([cur_dir, sep, 'evaluation_xvals.npy']))
p = np.load(''.join([cur_dir, sep, 'evaluation_pvals.npy']))

approx = []
real = []

fig1 = plt.figure()
color = ['k','g','b']
for i in range(0, len(layer_number)):
    output_file = ''.join(['layer_num_', str(layer_number[i]), '-neuron_number_',
                     str(neuron_number[i]), '-training_number_', str(training_number)])
    data = np.load(''.join([cur_dir, sep, output_file, '.npz']))
    p = data.f.p
    predict_approx=data.f.approx
    batch = data.f.batch
    error = data.f.error
    y = data.f.y
    ypts_real = data.f.ypts_real
    hist = data.f.history

    #### Print Figures ####
    if i == 1:
        real.append(plt.scatter(p, 1/np.cos(p), color = 'r', s = 4))
    approx.append(plt.scatter(p, predict_approx, color = color[i], s = 4))
    plt.title('Approximation of Secant(x) Using a Neural Network')
    plt.xlabel('x (radians)')
    plt.ylabel('sec(x)')
plt.legend((real[0], approx[0], approx[1], approx[2]), ('Analytical Value', 'Approx 2:20', 'Approx 4:10', 'Approx 8:5'))
fig1.savefig(''.join([cur_dir, sep, 'comparison_', 'layer_num_', str(layer_number), '-neuron_number_',
                         str(neuron_number), '-training_number_', str(training_number), '_multiple.svg']))

loss = []
fig2 = plt.figure()
for i in range(0, 3):
    output_file = ''.join(['layer_num_', str(layer_number[i]), '-neuron_number_',
                     str(neuron_number[i]), '-training_number_', str(training_number)])
    data = np.load(''.join([cur_dir, sep, output_file, '.npz']))
    p = data.f.p
    predict_approx = data.f.approx
    batch = data.f.batch
    error = data.f.error
    y = data.f.y
    ypts_real = data.f.ypts_real
    hist = data.f.history
    loss.append(plt.plot(range(0, len(error)), error))

plt.axis((-2.5, 100, -.1, 3))
plt.title('Neural Network Loss Progression with Training\nApproximation of Secant(x)')
plt.xlabel('Number of Training Iterations')
plt.ylabel('Mean Squared Error')
plt.legend((loss[0][0], loss[1][0], loss[2][0]), ('Loss 2:20', 'Loss 4:10', 'Loss 8:5'))
fig2.savefig(''.join([cur_dir, sep, 'error_', 'layer_num_', str(layer_number), '-neuron_number_',
                             str(neuron_number), '-training_number_', str(training_number), '_multiple.svg']))


# coding: utf-8

import re
from six import iteritems
from gensim import corpora, models

def get_dict(filepath, dictpath):
    f = open(filepath, "r", encoding="utf-8")
    dictionary = corpora.Dictionary(line.strip().split() for line in f)
    f.close()

    stop_words = [word.strip() for word in open("stop_words/stop_words.txt", "r", encoding="utf-8")]
    stop_ids = [dictionary.token2id[stopword] for stopword in stop_words if stopword in dictionary.token2id]
    non_zh_ids = [dictionary.token2id[word] for word,_ in dictionary.token2id.items() if re.match(r"[^\u4E00-\u9FA5]+", word)]
    once_ids = [tokenid for tokenid, docfreq in iteritems(dictionary.dfs) if docfreq == 1]
    dictionary.filter_tokens(stop_ids + once_ids + non_zh_ids)
    dictionary.compactify()
    print(dictionary)
    dictionary.save(dictpath)
    print("Save dictionary to %s for later use" % dictpath)

def _iter_(filepath, dictpath):
    dictionary = corpora.Dictionary.load(dictpath)
    f = open(filepath, "r", encoding="utf-8")
    for line in f:
        line = line.strip().split()
        yield dictionary.doc2bow(line)

def get_corpus(filepath, dictpath, corpuspath):
    corpus_iter = _iter_(filepath, dictpath)
    corpus_tf = [vector for vector in corpus_iter]
    print("Corpus example:%s" % corpus_tf[:5])
    corpora.MmCorpus.serialize(corpuspath, corpus_tf)
    print("Save corpus to %s for later use" % corpuspath)

def get_tfidf(dictpath, corpuspath, tfidfpath):
    dictionary = corpora.Dictionary.load(dictpath)
    print(dictionary)
    corpus_tf = corpora.MmCorpus(corpuspath)
    print(corpus_tf)
    tfidf_model = models.TfidfModel(corpus_tf)
    corpus_tfidf = tfidf_model[corpus_tf]
    corpora.MmCorpus.serialize(tfidfpath, corpus_tfidf)
    print("Save tfidf to %s for later use" % tfidfpath)

def get_lsi(dictpath, tfidfpath, lsipath, num_topics):
    dictionary = corpora.Dictionary.load(dictpath)
    print(dictionary)
    corpus_tfidf = corpora.MmCorpus(tfidfpath)
    print(corpus_tfidf)
    lsi = models.lsimodel.LsiModel(corpus=corpus_tfidf, id2word=dictionary, num_topics=num_topics)
    lsi.save(lsipath)
    print("Save lsi model to %s" % lsipath)

def get_lda(dictpath, tfidfpath, ldapath, num_topics):
    dictionary = corpora.Dictionary.load(dictpath)
    print(dictionary)
    corpus_tfidf = corpora.MmCorpus(tfidfpath)
    print(corpus_tfidf)
    lda = models.ldamodel.LdaModel(corpus=corpus_tfidf, id2word=dictionary, num_topics=num_topics, update_every=1, chunksize=10000, passes=1)
    lda.save(ldapath)
    print("Save lda model to %s" % ldapath)

def apply_labels(tfidfpath, ldapath, labelspath):
    corpus_tfidf = corpora.MmCorpus(tfidfpath)
    lda = models.LdaModel.load(ldapath)
    f = open(labelspath, "w", encoding="utf-8")
    for tfidf in corpus_tfidf:
        if tfidf:
            label,_ = sorted(lda[tfidf], key=lambda tup:tup[1], reverse=True)[0]
            f.write(str(label) + "\n")
        else:
            f.write("未分类" + "\n")
    f.close()
    print("Save label of documents to %s for classification" % labelspath)


from collections import defaultdict
from functools import partial
from itertools import permutations

from nltk.stem import PorterStemmer

def get_final_feature_descriptors(nltk_features, dep_features):
    stemmer = PorterStemmer()
    nltk_features_stemmed = [feat.split(' ') for feat in nltk_features]
    nltk_features_stemmed = map(partial(map, stemmer.stem), nltk_features_stemmed)
    for dep_feat in dep_features:
        dep_feat['token_stemmed'] = stemmer.stem(str(dep_feat['token']))
        dep_feat['compounds_stemmed'] = map(str, dep_feat['compounds'])
        dep_feat['compounds_stemmed'] = map(stemmer.stem, dep_feat['compounds_stemmed'])

    filtered = defaultdict(list)
    for nltk_feat, nltk_feat_stemmed in zip(nltk_features, nltk_features_stemmed):
        for dep_feat in dep_features:
            if nltk_feat_equals_dep_feat(nltk_feat_stemmed, dep_feat):
                filtered[nltk_feat] += dep_feat['descriptors']
    return filtered

def nltk_feat_equals_dep_feat(nltk_feat_stemmed, dep_feat):
    num_compounds = len(nltk_feat_stemmed) - 1
    for compounds_stemmed in permutations(dep_feat['compounds_stemmed'], num_compounds):
        compound_dep_feat = list(compounds_stemmed) + [dep_feat['token_stemmed']]
        if compound_dep_feat == nltk_feat_stemmed:
            return True
    return False

def dependency_features(doc):
    '''
    output ::= list<occurance>
    occurance ::= {'token': token, 'compunds': list<token>, 'descriptors': list<descriptor>}
    descriptor ::= {'token': token, 'negs': list<token>, 'advs': list<token>}
    '''

    descriptor_map = defaultdict(list)  # feat_index -> list<desc_index>
    compound_map = defaultdict(list)  # feat_index -> list<compound_index>
    neg_map = defaultdict(list)  # desc_index -> list<neg_index>
    adv_map = defaultdict(list)  # desc_index -> list<adv_index>

    for token in doc:
        if token.dep_ == 'amod':
            descriptor_map[token.head.i].append(token.i)
        elif token.dep_ in ['dobj', 'nmod', 'nsubj']:
            descriptor_map[token.i].append(token.head.i)
        elif token.dep_ == 'compound':
            compound_map[token.head.i].append(token.i)
        elif token.dep_ == 'neg':
            neg_map[token.head.i].append(token.i)
        elif token.dep_ == 'advmod':
            adv_map[token.head.i].append(token.i)

    occurances = []
    for feat_index, desc_indices in descriptor_map.iteritems():
        descriptors = []
        feat_negs = map(doc.__getitem__, neg_map[feat_index])
        for desc_index in desc_indices:
            negs = map(doc.__getitem__, neg_map[desc_index])
            advs = map(doc.__getitem__, adv_map[desc_index])
            descriptors.append({'token': doc[desc_index], 'negs': negs + feat_negs, 'advs': advs})
        compounds = map(doc.__getitem__, compound_map[feat_index])
        occurance = {'token': doc[feat_index], 'compounds': compounds, 'descriptors': descriptors}
        occurances.append(occurance)

    return occurances


from django.test import TestCase
from unittest.mock import patch

from trades.models import Currency, Trade

class TradeModelTest(TestCase):

    @classmethod
    def setUpTestData(cls):
        cls.unique_id = 'TR1234567'
        cls.sell_currency = Currency.objects.create(name='USD')
        cls.buy_currency = Currency.objects.create(name='EUR')
        Trade.objects.create(string_id=cls.unique_id, sell_currency=cls.sell_currency,
                             sell_amount=10, buy_currency=cls.buy_currency, buy_amount=15, rate=1.5)

    def test_id_uniqueness(self):
        available_id = '1234ASD'
        with patch('core.utils.generate_random_alphanumeric',
                   side_effect=[self.unique_id.strip('TR'), available_id]) as mock_id_randomness:
            Trade.objects.create(sell_currency=self.sell_currency, sell_amount=10,
                                 buy_currency=self.buy_currency, buy_amount=20, rate=2)
            self.assertEqual(mock_id_randomness.call_count, 2)
        self.assertEqual(Trade.objects.count(), 2)


#!/usr/bin/env python3
import csv
import sys
import re
import os
import glob
from typing import Dict, List, Tuple

from wisselwerking.history import Enrollment, EnrollmentCollection, read_history, rename_dept
from wisselwerking.settings import \
    capacity_file, \
    output_file, \
    ASSIGNED_CHOICE, \
    CAPACITY_CHOICE, \
    CAPACITY_VALUE, \
    TEAM, \
    MAIL_COLUMN, \
    ENROLLMENT_FIRSTNAME, \
    ENROLLMENT_LASTNAME, \
    ENROLLMENT_MAIL, \
    ENROLLMENT_DEPT, \
    ENROLLMENT_CHOICES, \
    RANDOM_CHOICE, \
    NONE_CHOICE, \
    NO_ASSIGNMENT

filename = sys.argv[1]
previous_years_dir = sys.argv[2]

#
# Start assigning!
#

enrollments: List[Dict[str, str]] = []
capacities = {}
counter = {}

assignments: List[Tuple[Dict[str, str], str]] = []

def format_name(enrollment, include_lastname=False):
    first_name = str.join(' ',
                          (part.capitalize() for part in enrollment[ENROLLMENT_FIRSTNAME].strip().split(' ')))
    if include_lastname:
        parts = []
        for part in enrollment[ENROLLMENT_LASTNAME].strip().split(' '):
            if part.lower() in ['van', 'von', 'de', 'der', 'den', 'die']:
                parts.append(part.lower())
            else:
                parts.append(part.capitalize())
        return first_name + ' ' + str.join(' ', parts)

    return first_name

def mail_template(assigned, enrollment):
    name = format_name(enrollment)
    first_choice = enrollment[ENROLLMENT_CHOICES[0]].strip()
    second_choice = enrollment[ENROLLMENT_CHOICES[1]].strip()
    third_choice = enrollment[ENROLLMENT_CHOICES[2]].strip()
    if assigned == first_choice or first_choice == RANDOM_CHOICE:
        # first choice
        content = f"""Je bent geplaatst voor de Wisselwerking {assigned}. We hebben je gegevens doorgegeven aan de contactpersoon van deze Wisselwerking. Deze zal contact met opnemen om verdere afspraken te maken over je deelname.

Heel veel plezier bij je wisselwerking!"""
    elif assigned == second_choice or assigned == third_choice or \
            RANDOM_CHOICE in [second_choice, third_choice]:
        # second, third choice
        if assigned == third_choice:
            ordinal = "derde"
        elif assigned == second_choice:
            ordinal = "tweede"
        else:
            # random!
            ordinal = "vrije"

        content = f"""Helaas was bij jouw eerste keuze voor de Wisselwerking bij {first_choice} geen plek meer. Je bent nu geplaatst bij je {ordinal} keuze: {assigned}.

We hebben je gegevens doorgegeven aan de contactpersoon van deze Wisselwerking. Deze zal contact met opnemen om verdere afspraken te maken over je deelname.

Heel veel plezier bij je wisselwerking!"""
    else:
        # nothing
        content = f"Je hebt je aangemeld voor de Wisselwerking {first_choice}. Helaas waren deze en eventuele verdere keuzes vol."

    return f"""
Beste {name},

{content}

Hartelijke groet,

Team Wisselwerking Geesteswetenschappen
{TEAM}
""".strip()

def get_capacity(choice) -> int:
    if choice == RANDOM_CHOICE:
        return 999
    try:
        capacity = capacities[choice]
        if capacity is None:
            raise KeyError
        return capacity
    except KeyError:
        while True:
            value = input(f"Capacity for {choice}? ")
            try:
                parsed = int(value)
                capacities[choice] = parsed
                return parsed
            except ValueError:
                print("NOPE")
                pass

def save_capacities():
    with open(capacity_file, mode="w", encoding="utf-8-sig") as csv_file:
        fieldnames = [CAPACITY_CHOICE, CAPACITY_VALUE]
        writer = csv.DictWriter(csv_file, fieldnames=fieldnames, delimiter=';')

        writer.writeheader()
        for (choice, value) in capacities.items():
            writer.writerow({
                CAPACITY_CHOICE: choice,
                CAPACITY_VALUE: value
            })

if os.path.exists(capacity_file):
    with open(capacity_file, mode="r", encoding="utf-8-sig") as csv_file:
        csv_reader = csv.DictReader(csv_file, delimiter=';')

        for row in csv_reader:
            try:
                capacity = int(row[CAPACITY_VALUE])
            except ValueError:
                capacity = None
            except TypeError:
                capacity = None
            capacities[row[CAPACITY_CHOICE]] = capacity

with open(filename, mode="r", encoding='iso8859-15') as csv_file:
    csv_reader = csv.DictReader(csv_file, delimiter=';')
    form_fieldnames = csv_reader.fieldnames
    line_count = 0

    for row in csv_reader:
        # The top row is the last entry
        enrollments.insert(0, row)
        line_count += 1

history = read_history(previous_years_dir)

# Make sure all possible choices are known
for enrollment in enrollments:
    for choice in map(lambda key: enrollment[key], ENROLLMENT_CHOICES):
        if not choice or choice.strip() in NONE_CHOICE:
            continue
        else:
            counter[choice.strip()] = 0

def assign_choice(enrollment: Dict[str, str], choice: str):
    try:
        counter[choice] += 1
    except KeyError:
        counter[choice] = 1
    assignments.append((enrollment, choice))
    unassigned.remove(enrollment)

    if choice in choices and counter[choice] >= get_capacity(choice):
        choices.remove(choice)

def show_historic_counts():
    print("""
    TOEWIJZINGEN VAN VORIGE WISSELWERKINGEN:
    """)
    historic_counts = {}
    for item in history.list_assigned():
        try:
            historic_counts[item] += 1
        except KeyError:
            historic_counts[item] = 1

    for item in sorted(historic_counts):
        print(f"{str(historic_counts[item]).rjust(3)} {item}")

def show_counts(show_unassigned=True):
    print("""
    AANTAL AANMELDINGEN:
    """)

    choices = sorted(set(capacities.keys()).union(counter.keys()))
    maxlength = sorted(len(choice) for choice in choices)[-1]
    sum = 0
    empty = []

    for choice in choices:
        count = counter.get(choice, 0)
        if count == 0:
            if choice != RANDOM_CHOICE:
                empty.append(choice)
        else:
            print(f"{str(count).rjust(3)} {choice}")
            sum += count

    print("=" * (maxlength + 4))
    print(f"{str(sum).rjust(3)} TOTAAL")

    if empty:
        print("""
    WISSELWERKINGEN ZONDER TOEWIJZINGEN:
    """)
        for choice in empty:
            print(choice)

    if show_unassigned and unassigned:
        print(f"""
    {len(unassigned)} WISSELWERKERS ZONDER TOEWIJZINGEN:
    """)
        for enrollment in unassigned:
            print(enrollment[ENROLLMENT_MAIL])

choices = list(sorted(set(capacities.keys()).union(counter.keys())))
for item, capacity in capacities.items():
    # possible to close a department
    if capacity == 0:
        choices.remove(item)
unassigned = list(enrollments)

priority = []
# Walk through the choices in iterations
# Give priority on order of choice and within that on order of enrollment
for key in ENROLLMENT_CHOICES:
    for enrollment in enrollments:
        choice = enrollment[key]
        if choice and choice not in NONE_CHOICE:
            priority.append((enrollment, choice.strip()))

try:
    while unassigned and choices:
        done = True
        for choice in list(choices):
            for enrollment, enrollment_choice in priority:
                if enrollment in unassigned and enrollment_choice == choice:
                    assign_choice(enrollment, choice)
                    done = False
                    break
        if done:
            break
except KeyboardInterrupt:
    # still store the updated capacities
    save_capacities()
    raise

# assign the random members
choices.remove(RANDOM_CHOICE)

def reassign_random(assignments: List[Tuple[Dict[str, str], str]], history: EnrollmentCollection):
    first = True
    for enrollment in list(enrollment for (enrollment, choice) in assignments if choice == RANDOM_CHOICE):
        unassigned.append(enrollment)
        email = enrollment[ENROLLMENT_MAIL]
        if first:
            print("\n\n===TUSSENSTAND===\n\n")
            show_historic_counts()
            show_counts(False)
            first = False
        previous = list(history.by_email(email))
        depts = set((rename_dept(row[ENROLLMENT_DEPT])) + map(lambda x: x.from_dept, previous))
        print(f"\n\n{email} ({'; '.join(depts)}) moet verrast worden")
        if previous:
            print("Deed eerder de volgende wisselwerkingen: " +
                  ", ".join(x.assigned_dept for x in previous ))
        else:
            print("Wisselwerking-newbie!")

        while True:
            choice = input("Wijs een andere wisselwerking toe: ")
            if (0 if choice not in counter else counter[choice]) < get_capacity(choice):
                for item in assignments:
                    check_enrolment, _ = item
                    if check_enrolment == enrollment:
                        assignments.remove(item)
                        break
                assign_choice(enrollment, choice)
                counter[RANDOM_CHOICE] -= 1
                break
            else:
                print("Zit al vol!")

reassign_random(assignments, history)

save_capacities()

show_counts()

# Store the assignments
with open(output_file, mode="w", encoding="utf-8-sig") as csv_file:
    fieldnames = [ASSIGNED_CHOICE, ENROLLMENT_MAIL, MAIL_COLUMN] + \
        [field for field in form_fieldnames if field != MAIL_COLUMN]
    writer = csv.DictWriter(csv_file, fieldnames=fieldnames, delimiter=';')

    writer.writeheader()
    for (row, assigned) in assignments:
        writer.writerow({
            ASSIGNED_CHOICE: assigned,
            "mail": mail_template(assigned, row),
            **row
        })

# Store the assignments per choice - to mail the organizers
output_prepath = str.join('.', output_file.split('.')[:-1])

def output_text_file(choice: str, escape=True):
    return output_prepath + '.' + (choice if not escape else re.sub(r'[\*\(\) \-\.\&]+', '-', choice)) + '.txt'

existing_files = glob.glob(output_text_file('*', False))

for choice in sorted(counter):
    if choice == RANDOM_CHOICE:
        continue
    count = counter[choice]
    target = output_text_file(choice)
    try:
        existing_files.remove(target)
    except ValueError:
        pass
    with open(target, mode="w", encoding="utf-8-sig") as txt_file:
        choice_assignments = []
        for (row, assigned) in assignments:
            if assigned == choice:
                choice_assignments.append(
                    f"{format_name(row, True)} <{row[ENROLLMENT_MAIL]}> ({rename_dept(row[ENROLLMENT_DEPT])})\n")

        if count > 0:
            txt_file.writelines([f"""Beste organisator,

Leuk dat je je hebt opgegeven om een wisselwerking te organiseren! Voor de wisselwerking {choice} hebben de volgende {count} person(en) zich aangemeld:

"""] +
                                choice_assignments +
                                [f"""
Zou je zo snel mogelijk contact willen opnemen met deze mensen om afspraken te maken over de wisselwerking?

Heel veel plezier bij de wisselwerking!

Hartelijke groet,

Team Wisselwerking Geesteswetenschappen
{TEAM}
"""])
        else:
            txt_file.writelines([f"""Beste organisator,

Helaas heeft dit jaar niemand zich aangemeld voor de Wisselwerking {choice}.

Dank dat je een Wisselwerking wilde organiseren. We hopen dat je volgend jaar weer meedoet!

Hartelijke groet,

Team Wisselwerking Geesteswetenschappen
{TEAM}
"""])

# Clear existing files, which are no longer assigned
for existing_file in existing_files:
    os.remove(existing_file)

print("DONE! Plaats toewijzingen.csv op de O-schijf")


"""Tests standard tap features using the built-in SDK tests library."""

import json
from pathlib import Path

import jsonschema
import pytest
from singer_sdk.testing import get_standard_tap_tests

from tap_atinternet.tap import TapATInternet

with open(Path(__file__).parents[2] / ".secrets/config.gn.json") as f:
    SAMPLE_CONFIG_GN = json.load(f)
with open(Path(__file__).parents[2] / ".secrets/config.sp.json") as f:
    SAMPLE_CONFIG_SP = json.load(f)
with open(Path(__file__).parent / "incremental_catalog.json") as f:
    # you can obtain this file by configuring the tap to incremental sync, and running
    # `meltano elt tap-atinternet target-jsonl --dump catalog`
    INCREMENTAL_CATALOG = json.load(f)

@pytest.fixture
def configs():
    return [SAMPLE_CONFIG_GN, SAMPLE_CONFIG_SP]

# Run standard built-in tap tests from the SDK:
def test_standard_tap_tests(configs):
    """Run standard tap tests from the SDK."""
    for config in configs:
        tests = get_standard_tap_tests(TapATInternet, config=config)
        for test in tests:
            test()

# Additional tests
def test_incremental_sync(configs):
    for config in configs:
        tap = TapATInternet(config=config, catalog=INCREMENTAL_CATALOG)
        tap.run_connection_test()

def test_record_schema(configs):
    """
    Validate the tap records against the provided schema.

    This should part of the SDK standard tests (above), but this not the case yet
    (see https://gitlab.com/meltano/sdk/-/issues/228).

    Note: During ELT pipelines, target can be configured to validate the input records against
    the schema, but this is not an obligation.
    """
    for config in configs:
        tap = TapATInternet(config=config)
        tap.run_connection_test()  # start with this to initialize stream_state

        streams = tap.discover_streams()
        for stream in streams:
            records_iterator = stream.get_records(context=None)
            first_record = next(records_iterator)
            print('Record:\n', first_record)
            print('Schema:\n', stream.schema)
            jsonschema.validate(instance=first_record, schema=stream.schema)