import copy
from enum import IntEnum
from typing import List, Optional, Sequence, Tuple

from tetris.constants import (
    BOARD_HEIGHT,
    BOARD_WIDTH,
    LINE_CLEAR_SCORES,
    BoardState,
    Cell,
)
from tetris.piece import Piece

class Action(IntEnum):
    NOOP = 0
    ROTATE_ANTICLOCKWISE = 1
    ROTATE_CLOCKWISE = 2
    MOVE_LEFT = 3
    MOVE_RIGHT = 4
    HARD_DROP = 5

class Board:
    board: BoardState
    piece: Optional[Piece]
    running: bool

    def __init__(self) -> None:
        self.board = [[None] * BOARD_WIDTH for _ in range(BOARD_HEIGHT)]
        self.piece = None

    def set_piece(self, piece: Piece) -> None:
        """Sets a piece to be the current piece being controlled.

        Args:
            piece (Piece): A Tetris piece.
        """

        self.piece = piece

    def is_game_running(self) -> bool:
        """Checks the board to see if there are any cells filled in the top playable row.
           If there are, the player has lost the game.

        Returns:
            bool: False if the board has any pieces in the top row else True.
        """

        return self.board[1].count(None) == BOARD_WIDTH

    def copy(self) -> BoardState:
        """Copies the internal board state.

        Returns:
            BoardState: The board.
        """

        return copy.deepcopy(self.board)

    def get_changes(self, old_board: BoardState) -> List[Tuple[str, int, int]]:
        """Identifies differences between the current board and another board state.

        Args:
            old_board (BoardState): The board to compare against.

        Returns:
            List[Tuple[str, int, int]]: A list of changes in the form (cell_value, y, x).
        """

        return [
            ("N" if not self.board[i][j] else self.board[i][j], i, j)
            for i, row in enumerate(old_board)
            for j, piece_type in enumerate(row)
            if self.board[i][j] != piece_type
        ]

    def find_lines_to_clear(self) -> List[int]:
        """Iterates through each row in the board to see if it's full and can be cleared.

        Returns:
            List[int]: Indices of each row that needs to be cleared.
        """

        return [i for i in reversed(range(BOARD_HEIGHT)) if None not in self.board[i]]

    def clear_lines(self, line_indices: List[int]) -> int:
        """Removes each row to be cleared from the board and adds empty rows to the top.
           Gets the score based on how many rows have been cleared.

        Args:
            line_indices (List[int]): The list of index rows to be cleared.

        Returns:
            int: The score that the number of cleared lines are worth.
        """

        if not line_indices:
            return 0

        for i in sorted(line_indices, reverse=True):
            del self.board[i]

        self.board = [
            [None] * BOARD_WIDTH for _ in range(len(line_indices))
        ] + self.board

        return LINE_CLEAR_SCORES[len(line_indices)]

    def update_board(self, old: List[Cell], new: List[Cell]) -> None:
        """Updates the board based on old and new locations of the current piece.

        Lists can be empty if no changes.

        Args:
            old (List[Cell]): The old (y, x) coordinates of the piece to be removed.
            new (List[Cell]): The new (y, x) coordinates of the piece to be added.
        """

        for old_cell in old:
            if 0 <= old_cell[0] < BOARD_HEIGHT and 0 <= old_cell[1] < BOARD_WIDTH:
                self.board[old_cell[0]][old_cell[1]] = None
            else:
                raise RuntimeError("Cell out of bounds of the board.")

        for new_cell in new:
            if 0 <= new_cell[0] < BOARD_HEIGHT and 0 <= new_cell[1] < BOARD_WIDTH:
                self.board[new_cell[0]][new_cell[1]] = self.piece.piece_type
            else:
                raise RuntimeError("Cell out of bounds of the board.")

    def spawn_piece(self) -> bool:
        """Spawns the board object's current piece on the board.

        If this piece has spawned inside another piece, set it to landed.

        Returns:
            bool: Whether the piece could be spawned successfully
        """

        new_piece_location = self.piece.spawn_piece()
        for coord in new_piece_location:
            if self.board[coord[0]][coord[1]]:
                self.piece.landed = True
                return False

        self.update_board([], new_piece_location)

        return True

    def move_piece_left(self) -> None:
        """Moves the current piece left and updates board if required."""

        old, new = self.piece.move_left(self.board)

        if (old is None) ^ (new is None):
            raise RuntimeError("Error computing piece move left.")

        if old and new:
            self.update_board(old, new)

    def move_piece_right(self) -> None:
        """Moves the current piece right and updates board if required."""

        old, new = self.piece.move_right(self.board)

        if (old is None) ^ (new is None):
            raise RuntimeError("Error computing piece move right.")

        if old and new:
            self.update_board(old, new)

    def rotate_piece_clockwise(self) -> None:
        """Rotates the current piece clockwise and updates board if required."""

        old, new = self.piece.rotate_clockwise(self.board)

        if (old is None) ^ (new is None):
            raise RuntimeError("Error computing piece rotate clockwise.")

        if old and new:
            self.update_board(old, new)

    def rotate_piece_anticlockwise(self) -> None:
        """Rotates the current piece anticlockwise and updates board if required."""

        old, new = self.piece.rotate_anticlockwise(self.board)

        if (old is None) ^ (new is None):
            raise RuntimeError("Error computing piece rotate anticlockwise.")

        if old and new:
            self.update_board(old, new)

    def fall(self) -> None:
        """Drops the current piece one position if it has not yet landed and updates the board if required."""

        self.piece.landed = self.piece.has_landed(self.board)
        if not self.piece.landed:
            old, new = self.piece.fall()

            if (old is None) ^ (new is None):
                raise RuntimeError("Error computing piece fall.")

            self.update_board(old, new)

    def apply_action(self, action: Action) -> None:
        """Applies an action to the current board.

        Args:
            action (Action): The action to apply.
        """

        if action == Action.HARD_DROP:
            while not self.piece.landed:
                self.fall()
            return
        elif action == Action.ROTATE_ANTICLOCKWISE:
            self.rotate_piece_anticlockwise()
        elif action == Action.ROTATE_CLOCKWISE:
            self.rotate_piece_clockwise()
        elif action == Action.MOVE_LEFT:
            self.move_piece_left()
        elif action == Action.MOVE_RIGHT:
            self.move_piece_right()

        self.fall()

    def with_move(self, action: Action) -> "Board":
        """Copies and returns the current board with a single action applied.

        Args:
            action (Action): The action to apply.

        Returns:
            Board: A copy of the current board with the action applied.
        """

        return self.with_moves([action])

    def with_moves(self, actions: Sequence[Action]) -> "Board":
        """Copies and returns the current board with the sequence of actions applied.

        Args:
            actions (List[Action]): A list of actions to perform.

        Returns:
            Board: A copy of the current board with the actions applied.
        """

        board = copy.deepcopy(self)

        for action in actions:
            board.apply_action(action)

            if board.piece.landed:
                break

        return board

    def __getitem__(self, y):
        """Accesses board rows."""

        return self.board[y]


from PIL import Image
import numpy
import sys, os, json
import glob
import piexif
import shutil

def get_exif_files(path):
    upload_path = sys.path[0] + path

    # get all files
    files = []
    for (dirpath, dirnames, filenames) in os.walk(upload_path):
        files.extend(filenames)
        break

    # build history
    items = []
    for file in files:
        file_path = os.path.join(upload_path, file)
        exif_dict = piexif.load(file_path)

        usercomment = exif_dict["Exif"][37510].decode("utf-8")
        usercomment = json.loads(usercomment)

        items.append(usercomment)

    return items

def get_exif_file(path, file_name):
    upload_path = sys.path[0] + path
    file_path = os.path.join(upload_path, file_name)
    try:
        exif_dict = piexif.load(file_path)

        usercomment = exif_dict["Exif"][37510].decode("utf-8")
        usercomment = json.loads(usercomment)

        return usercomment
    except:
        return None

def save_exif_file(userdata):
    path = userdata["path"]
    userdata = json.dumps(userdata)
    exif_ifd = {piexif.ExifIFD.UserComment: userdata}
    exif_dict = {"Exif": exif_ifd}
    exif_bytes = piexif.dump(exif_dict)

    im = Image.open(path)
    im.save(path, exif=exif_bytes)

def remove_file(path, file_name=None, is_complete=True):
    if is_complete:
        os.remove(path)
    else:
        path = sys.path[0] + path
        path = os.path.join(path, file_name)
        os.remove(path)

def insert_or_delete_file(userdata, destination_path):
    file_name = userdata["id"]
    original_path = userdata["path"]
    upload_path = sys.path[0] + destination_path
    upload_path = os.path.join(upload_path, file_name)

    if os.path.isfile(upload_path):
        os.remove(upload_path)
    else:
        shutil.copy2(original_path, upload_path)

def save_file(path, file):
    output = open(path, 'wb')
    output.write(file)
    output.close()

def get_images_as_byte_array(path, extension):
    download_path = sys.path[0] + path + "/*" + extension
    image_list = []
    for filename in glob.glob(download_path):
        with open(filename, "rb") as imageFile:
            f = imageFile.read()
            b = bytearray(f)
            image_list.append(b)

    return image_list

def get_image(path, resized=True):
    print("downloading file")
    pil_image = Image.open(path).convert('RGB')

    # resize image if needed
    if resized:
        w, h = pil_image.size[0], pil_image.size[1]
        if w > 640:
            pil_image = __resize_image_basewidth(pil_image, 640)
            h = pil_image.size[1]
        if h > 480:
            pil_image = __resize_image_baseheight(pil_image, 640)

    open_cv_image = numpy.array(pil_image)
    open_cv_image = open_cv_image[:, :, ::-1].copy()

    return open_cv_image

def __resize_image_basewidth(source, basewidth):
    wpercent = (basewidth / float(source.size[0]))
    hsize = int((float(source.size[1])*float(wpercent)))
    img = source.resize((basewidth, hsize), Image.ANTIALIAS)
    return img

def __resize_image_baseheight(source, baseheight):
    hpercent = (baseheight / float(source.size[1]))
    wsize = int((float(source.size[0])*float(hpercent)))
    img = source.resize((baseheight, wsize), Image.ANTIALIAS)
    return img


import datetime
from typing import List, Optional

import numpy as np
from ..common import (
    ActionType, Context, DataSource, Processor, ProcessorFactory, TradingFrequency,
    Position, ProcessorAction, Mode, DATETIME_TYPE, DAYS_IN_A_QUARTER)
from ..stock_universe import IntradayVolatilityStockUniverse

NUM_UNIVERSE_SYMBOLS = 20
EXIT_TIME = datetime.time(16, 0)
# 3 hours only works if 1 hour and 2 hours are not triggered
PARAMS = [(10, 1), (13, 1), (25, 1.75), (30, 2.25), (37, 2.25)]

class H2lHourProcessor(Processor):

    def __init__(self,
                 lookback_start_date: DATETIME_TYPE,
                 lookback_end_date: DATETIME_TYPE,
                 data_source: DataSource,
                 output_dir: str) -> None:
        super().__init__(output_dir)
        self._positions = dict()
        self._stock_universe = IntradayVolatilityStockUniverse(lookback_start_date,
                                                               lookback_end_date,
                                                               data_source,
                                                               num_stocks=NUM_UNIVERSE_SYMBOLS)

    def get_trading_frequency(self) -> TradingFrequency:
        return TradingFrequency.FIVE_MIN

    def setup(self, hold_positions: List[Position], current_time: Optional[DATETIME_TYPE]) -> None:
        to_remove = [symbol for symbol, position in self._positions.items()
                     if position['status'] != 'active']
        for symbol in to_remove:
            self._positions.pop(symbol)

    def get_stock_universe(self, view_time: DATETIME_TYPE) -> List[str]:
        return list(set(self._stock_universe.get_stock_universe(view_time) +
                        list(self._positions.keys())))

    def process_data(self, context: Context) -> Optional[ProcessorAction]:
        if self.is_active(context.symbol):
            return self._close_position(context)
        elif (context.symbol not in self._positions or
              self._positions[context.symbol]['status'] == 'pending'):
            return self._open_position(context)

    def _open_position(self, context: Context) -> Optional[ProcessorAction]:
        t = context.current_time.time()
        if t >= EXIT_TIME:
            return
        interday_closes = context.interday_lookback['Close']
        # Avoid short squeeze caused by retail trader
        if max(interday_closes[-1], context.current_price) / interday_closes[-5] > 4:
            return
        market_open_index = context.market_open_index
        if market_open_index is None:
            return
        intraday_closes = context.intraday_lookback['Close'][market_open_index:]
        if len(intraday_closes) < 10:
            return
        if context.current_price > np.min(intraday_closes):
            return
        if abs(context.current_price / context.prev_day_close - 1) > 0.5:
            return
        intraday_opens = context.intraday_lookback['Open'][market_open_index:]
        if intraday_opens[-1] > context.prev_day_close > intraday_closes[-1]:
            return
        if interday_closes[-1] > 10 * np.min(interday_closes[-DAYS_IN_A_QUARTER:]):
            return
        h2l_avg = context.h2l_avg
        h2l_std = context.h2l_std
        lower_threshold = max(h2l_avg - 3 * h2l_std, -0.5)
        for n, z in PARAMS:
            if len(intraday_closes) < n:
                continue
            current_loss = context.current_price / intraday_closes[-n] - 1
            z0 = 0.1 if t < datetime.time(11, 0) or t > datetime.time(15, 0) else 0
            upper_threshold = h2l_avg - (z + z0) * h2l_std
            is_trade = lower_threshold < current_loss < upper_threshold
            if is_trade or (context.mode == Mode.TRADE and current_loss < upper_threshold * 0.8):
                self._logger.debug(f'[{context.current_time.strftime("%F %H:%M")}] [{context.symbol}] '
                                   f'Current loss: {current_loss * 100:.2f}%. N: {n}. '
                                   f'Threshold: {lower_threshold * 100:.2f}% ~ {upper_threshold * 100:.2f}%. '
                                   f'Current price {context.current_price}.')
            if is_trade:
                self._positions[context.symbol] = {'entry_time': context.current_time,
                                                   'status': 'pending', 'n': n}
                return ProcessorAction(context.symbol, ActionType.BUY_TO_OPEN, 1)

    def _close_position(self, context: Context) -> Optional[ProcessorAction]:
        position = self._positions[context.symbol]
        intraday_closes = context.intraday_lookback['Close']
        index = -position['n'] - (context.current_time - position['entry_time']).seconds // 300
        stop_loss = False
        if len(intraday_closes) >= abs(index):
            current_loss = context.current_price / intraday_closes[index] - 1
            lower_threshold = max(context.h2l_avg - 3 * context.h2l_std, -0.5)
            stop_loss = current_loss < lower_threshold
        if (stop_loss or
                context.current_time >= position['entry_time'] + datetime.timedelta(minutes=35) or
                context.current_time.time() >= EXIT_TIME):
            self._logger.debug(f'[{context.current_time.strftime("%F %H:%M")}] [{context.symbol}] '
                               f'Closing position. Current price {context.current_price}.')
            position['status'] = 'inactive'
            return ProcessorAction(context.symbol, ActionType.SELL_TO_CLOSE, 1)

class H2lHourProcessorFactory(ProcessorFactory):

    def __init__(self):
        super().__init__()

    def create(self,
               lookback_start_date: DATETIME_TYPE,
               lookback_end_date: DATETIME_TYPE,
               data_source: DataSource,
               output_dir: str,
               *args, **kwargs) -> H2lHourProcessor:
        return H2lHourProcessor(lookback_start_date, lookback_end_date, data_source, output_dir)


import preprocess
import recognize
import printed_icr
from pdf2image import convert_from_path

def intel_char_recog(filename, texttype, filetype):
    output = 'default - empty string'
    if texttype.lower() == 'print':
        if filetype.lower() == 'pdf':
            pages = convert_from_path(filename, 500)
            # for page in pages:
            page[0].save('image.jpg', 'JPEG')
            output = printed_icr.printed_intel_char_recog('image.jpg')
        elif filetype.lower() == 'image':
            output = printed_icr.printed_intel_char_recog(filename)
    elif texttype.lower() == 'handwritten':
        if filetype.lower() == 'image':
            folder = preprocess.save_words_and_lines(filename)
            output = recognize.recog(folder)
    return output

def main():
    print(intel_char_recog('sample_segmentation_data/png1.jpg', 'handwritten', 'image'))

if __name__ == '__main__':
    main()


import os
import sys
import getpass
import subprocess
from subprocess import CalledProcessError
import shlex
from datetime import datetime
try:
    import configparser
except ImportError:
    import ConfigParser as configparser

# logging levels
WARN = 'WARN'
INFO = 'INFO'
ERROR = 'ERROR'
DEBUG = 'DEBUG'

PIPE = subprocess.PIPE

def log(message, name, level, write_location):
    # timestamp - [level] - [pipelinename notifiername script] - log message
    log_message = (str(datetime.utcnow()) + ' - [' + level + '] - [' +
                   name + '] - ' + message)

    now = datetime.utcnow()
    log_file = str(now.date()) + ' ' + str(now.hour) + 'h.log'

    __write_to_log_file(log_message, write_location, log_file)

def __write_to_log_file(message, write_dir, log_file_name):
    if write_dir.endswith('/'):
        write_dir = write_dir[:-1]

    log_file = open(write_dir + '/' + log_file_name, 'a+')
    log_file.write(message + '\n')
    log_file.close()

def print_stderr(output):
    sys.stderr.write(output + '\n')

def list_directory(directory, file_extension=None):
    files = []

    if os.path.isdir(directory):
        files = os.listdir(directory)
    else:
        log("Directory: " + directory + " is not a valid directory.", level=ERROR)

    if file_extension:
        files = filter(lambda x: x.endswith(file_extension), files)

    return files

def run_command(command):
    # see http://stackoverflow.com/questions/6466711/what-is-the-return-value-of-os-system-in-python/35362488#35362488
    return os.system(command) >> 8

def run_command_async(command):
    process = __get_last_process_in_chain(command)

def capture_command_output(command):
    return_code, stdout, stderr = 1, '', ''

    try:
        process = __get_last_process_in_chain(command)
        stdout, stderr = tuple(process.communicate())
        return_code = process.returncode
    except OSError as e:
        stderr = str(e)
        return_code = e.errno
    except CalledProcessError as e:
        stderr = e.output
        return_code = e.return_code

    stdout = str(stdout.decode('utf-8'))
    stderr = str(stderr.decode('utf-8'))

    return return_code, stdout, stderr

# If there are pipes in the command, multiple process objects must be made to avoid
# the shell=True option in the Popen constructor.
def __get_last_process_in_chain(command_string):
    command_words = shlex.split(command_string)

    command_list = []
    current_command_index = 0

    # If command_string is 'ps aux | grep root | grep /usr' this loop
    # would result in [['ps', 'aux'], ['grep', 'root'], ['grep', '/usr']]
    for command_word in command_words:
        if command_word != '|':
            if len(command_list) <= current_command_index:
                command_list.append([])

            command_list[current_command_index].append(command_word)
        else:
            current_command_index += 1

    process_list = []
    process_list.append(subprocess.Popen(command_list[0],
                                         stdout=PIPE, stderr=PIPE))

    for i in range(1, len(command_list)):
        command = command_list[i]
        process_list.append(subprocess.Popen(command, stdout=PIPE,
                            stdin = process_list[i - 1].stdout, stderr=PIPE))

    return process_list[-1]

def build_hive_command(hql_path, parameters):
    param_string = ''
    for key in parameters:
        param_string += ' --hiveconf ' + str(key) + '=' + str(parameters[key])

    return 'hive -f ' + hql_path + param_string

def load_config(config_file_path, config=None):
    if not config:
        config = configparser.ConfigParser()

    config.read(config_file_path)

    return config

def get_user():
    return getpass.getuser()

def get_project_root_dir():
    # Gets "realpath" of this script and returns the path three levels up (the project root directory)
    return os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))


import cv2 as cv
import sys

img = cv.imread(cv.samples.findFile("aisle.jpg"))

if img is None:
    sys.exit("Image not found.")

cv.imshow("Display Window", img)
# waits a key from keyboard and stores in k
k = cv.waitKey(0)

# if users press the s , imwrite saves image as png
if k == ord("s"):
    cv.imwrite("aisle.png", img)

cv.destroyWindow()


from django.db import models
from _lib import panda
from django.db.models import Max
from django.core.validators import MaxValueValidator, MinValueValidator

from _mc_subdept.models import Subdept
# Create your models here.
class Category(models.Model):
    deptcode = models.CharField(db_column='DeptCode', max_length=6)  # Field name made lowercase.
    mcode = models.ForeignKey(Subdept, models.DO_NOTHING, db_column='MCode',related_name='category_key')  # Field name made lowercase.
    code = models.CharField(db_column='Code', primary_key=True, max_length=6,editable=False)  # Field name made lowercase.
    description = models.CharField(db_column='Description', max_length=40)  # Field name made lowercase.
    margin_min = models.DecimalField(max_digits=5, decimal_places=2, validators=[MinValueValidator(0.00)])
    margin_max = models.DecimalField(max_digits=5, decimal_places=2, validators=[MinValueValidator(0.00)])
    tax_code_purchase = models.CharField(max_length=10)
    tax_code_supply = models.CharField(max_length=10)
    margin = models.DecimalField(max_digits=5, decimal_places=2, validators=[MinValueValidator(0.00)])
    created_at = models.DateTimeField(db_column='CREATED_AT', null=True,editable=False)  # Field name made lowercase.
    created_by = models.CharField(db_column='CREATED_BY', max_length=30,editable=False)  # Field name made lowercase.
    updated_at = models.DateTimeField(db_column='UPDATED_AT', blank=True, null=True,editable=False)  # Field name made lowercase.
    updated_by = models.CharField(db_column='UPDATED_BY', max_length=30)  # Field name made lowercase.

    class Meta:
        managed = False
        db_table = 'category'
        ordering = ('code',)

    def __str__(self):
        return self.code

    def get_absolute_url(self):
        return f'/{self.code}/'

    # def next_running_number:
    #     maxCode = Category.objects.all().aggregate(Max('code'))
    #     if maxCode=='':
    #         return 1
    #     else:
    #         return maxCode+1

    def save(self, *args, **kwargs):
        print('1234567')
        print('self_group:',self.code)
        if self.created_by =='':
            maxCode = Category.objects.all().aggregate(Max('code'))
            if maxCode=='':
                next_running_number = 1
            else:
                print('print maxcode', maxCode)
                print(type(maxCode))
                next_running_number = int(maxCode['code__max'])+1
            self.code='{:04}'.format(next_running_number)
            self.created_at=panda.panda_today()
            self.created_by=self.updated_by
            self.updated_at=panda.panda_today()
        else:
            self.updated_at=panda.panda_today()
        super(Category, self).save(*args, **kwargs)


"""Tests for search tasks."""

from unittest import mock
import pytest

from django.urls import reverse
from django.utils import timezone

from readthedocs.search.models import SearchQuery
from readthedocs.search import tasks

@pytest.mark.django_db
@pytest.mark.search
@pytest.mark.usefixtures("all_projects")
class TestSearchTasks:
    @classmethod
    def setup_class(cls):
        # This reverse needs to be inside the ``setup_class`` method because from
        # the Corporate site we don't define this URL if ``-ext`` module is not
        # installed
        cls.url = reverse("search_api")

    def test_search_query_recorded_when_results_not_zero(self, api_client):
        """Test if search query is recorded in a database when a search is made."""

        assert (
            SearchQuery.objects.all().count() == 0
        ), "no SearchQuery should be present if there is no search made."

        # `sphinx` is present in `documentation.json`
        # file of project `kuma`
        search_params = {"q": "sphinx", "project": "kuma", "version": "latest"}
        resp = api_client.get(self.url, search_params)

        assert resp.data["count"] == 1
        assert (
            SearchQuery.objects.all().count() == 1
        ), "there should be 1 obj since a search is made which returns one result."

    def test_partial_queries_are_not_recorded(self, api_client):
        """Test if partial queries are not recorded."""

        assert (
            SearchQuery.objects.all().count() == 0
        ), "no SearchQuery should be present if there is no search made."

        time = timezone.now()
        search_params = {"q": "stack", "project": "docs", "version": "latest"}

        with mock.patch("django.utils.timezone.now") as test_time:
            test_time.return_value = time
            resp = api_client.get(self.url, search_params)
            assert resp.status_code, 200

        assert (
            SearchQuery.objects.all().count() == 1
        ), "one SearchQuery should be present"

        # update the time and the search query and make another search request
        time = time + timezone.timedelta(seconds=2)
        search_params["q"] = "stack over"
        with mock.patch("django.utils.timezone.now") as test_time:
            test_time.return_value = time
            resp = api_client.get(self.url, search_params)
            assert resp.status_code, 200

        # update the time and the search query and make another search request
        time = time + timezone.timedelta(seconds=2)
        search_params["q"] = "stack overflow"
        with mock.patch("django.utils.timezone.now") as test_time:
            test_time.return_value = time
            resp = api_client.get(self.url, search_params)
            assert resp.status_code, 200

        assert (
            SearchQuery.objects.all().count() == 1
        ), "one SearchQuery should be present"
        assert (
            SearchQuery.objects.all().first().query == "stack overflow"
        ), "one SearchQuery should be there because partial queries gets updated"

    def test_search_query_recorded_when_results_are_zero(self, api_client):
        """Test that search queries are recorded when they have zero results."""

        assert (
            SearchQuery.objects.all().count() == 0
        ), "no SearchQuery should be present if there is no search made."

        # `readthedo` is NOT present in project `kuma`.
        search_params = {"q": "readthedo", "project": "kuma", "version": "latest"}
        resp = api_client.get(self.url, search_params)

        assert resp.data["count"] == 0
        assert SearchQuery.objects.all().count() == 1

    def test_delete_old_search_queries_from_db(self, project):
        """Test that the old search queries are being deleted."""

        assert (
            SearchQuery.objects.all().count() == 0
        ), "no SearchQuery should be present if there is no search made."

        obj = SearchQuery.objects.create(
            project=project, version=project.versions.all().first(), query="first"
        )
        obj.created = timezone.make_aware(timezone.datetime(2019, 1, 1))
        obj.save()

        assert SearchQuery.objects.all().count() == 1
        tasks.delete_old_search_queries_from_db()
        assert SearchQuery.objects.all().count() == 0


from data.external_data import CovidData

from pyomo.environ import *
import numpy as np
from pyomo.opt import SolverFactory
import pyomo.environ

class TransOpt:

    def __init__(self, data):
        self.data = data

    def create_model(self, df_cases, time_horizon=28,start_day=120, n1=20000, n2=300, n3=100, n4=150):

        def haversine_np(lon1, lat1, lon2, lat2):
            """
            Calculate the great circle distance between two points
            on the earth (specified in decimal degrees)

            All args must be of equal length.

            """
            lon1, lat1, lon2, lat2 = map(np.radians, [lon1, lat1, lon2, lat2])

            dlon = lon2 - lon1
            dlat = lat2 - lat1

            a = np.sin(dlat/2.0)**2 + np.cos(lat1) * np.cos(lat2) * np.sin(dlon/2.0)**2

            c = 2 * np.arcsin(np.sqrt(a))
            miles = 3959 * c
            return miles

        df_geo = self.data.df_geo.copy().reset_index()
        dc_geo = self.data.dc_geo.copy().reset_index()
        production_geo = self.data.production_geo.copy().reset_index()

        temp = dc_geo.assign(A=1).merge(df_geo.assign(A=1), on='A').drop('A', 1)
        temp['distance'] = haversine_np(temp['longitude_x'], temp['latitude_x'], temp['longitude_y'], temp['latitude_y'])
        distance_matrix_demand = temp.set_index(['FIPS_x', 'FIPS_y'])['distance'].to_dict()

        temp = production_geo.assign(A=1).merge(dc_geo.assign(A=1), on='A').drop('A', 1)
        temp['distance'] = haversine_np(temp['longitude_x'], temp['latitude_x'], temp['longitude_y'], temp['latitude_y'])
        distance_matric_production = temp.set_index(['FIPS_x', 'FIPS_y'])['distance'].to_dict()

        temp = dc_geo.assign(A=1).merge(dc_geo.assign(A=1), on='A').drop('A', 1)
        temp['distance'] = haversine_np(temp['longitude_x'], temp['latitude_x'], temp['longitude_y'], temp['latitude_y'])
        distance_matrix_dc = temp.set_index(['FIPS_x', 'FIPS_y'])['distance'].to_dict()

        miles_per_day = 650
        transit_time_dc_dem = {k:int(v/miles_per_day) + 1 for k,v in distance_matrix_demand.items()}
        transit_time_dc_dc = {k:int(v/miles_per_day) + 1 for k,v in distance_matrix_dc.items()}
        transit_time_prod_dc = {k:int(v/miles_per_day) + 1 for k,v in distance_matric_production.items()}
        # Creation of a Concrete Model
        self.model = ConcreteModel()

        holding_capacities = {'47137':160000,'56027':60000, '41033': 25000, '39109': 120000}
        start_capacities = {'47137':n1,'56027':n2, '41033': n3, '39109': n4}
        production_capacity = {'13103':500000,'26119':250000, '38027': 100000}
        shipping_times = None

        self.model.dcs = Set(initialize=holding_capacities.keys(), doc='DC')
        self.model.demand_nodes = Set(initialize=df_geo['FIPS'], doc='Demand_nodes')
        self.model.production_nodes = Set(initialize=production_capacity.keys())
        self.model.t = Set(initialize=range(time_horizon), doc='horizon')

        demand = dict()
        for k,v in df_cases.iloc[start_day:start_day + time_horizon].reset_index().drop(columns = 'date').to_dict().items():
            for kk, vv in v.items():
                demand[k,kk] = vv

        self.model.holding_capacities = Param(self.model.dcs, initialize=holding_capacities, doc='Holding Capacity of DC i in cases per day')
        self.model.production_capacities = Param(self.model.production_nodes, initialize=production_capacity, doc='Holding Capacity of DC i in cases per day')

        self.model.demand = Param(self.model.demand_nodes, self.model.t, initialize=demand, doc='Demand at market j in cases')

        self.model.dist_mat_dem = Param(self.model.dcs,self.model.demand_nodes, initialize=distance_matrix_demand, doc='Distance in miles')
        self.model.dist_mat_dc = Param(self.model.dcs,self.model.dcs, initialize=distance_matrix_dc, doc='Distance in miles from dc to dc')
        self.model.dist_mat_prod_dc = Param(self.model.production_nodes, self.model.dcs, initialize= distance_matric_production, doc = 'distacne in miled to prod plants to DCs')
        #  Scalar f  freight in dollars per case per thousand miles  /90/ ;
        self.model.freight_cost_dc_dem = Param(initialize=0.01, doc='Freight in dollars per case per mile from a dc to a demand node')
        self.model.freight_cost_dc_dc = Param(initialize = 0.002, doc = 'Freight cost from dc to dc in dollars per case per mile')
        self.model.storage_cost = Param(initialize=0.001, doc = 'cost of storing case at dc')
        self.model.production_cost = Param(initialize = 0.009, doc = 'production price per unit')

        def cost_init(model, i, j):
            return model.freight_cost_dc_dem * (model.dist_mat_dem[i,j] + 10.0) / 1000.0

        def cost_dc(model,i,j):
            if i != j:
                val = model.freight_cost_dc_dc* (model.dist_mat_dc[i,j] + 10.0)/1000.0
                return val
            else:
                return 1000.0

        def cost_prod_dc(mode, i ,j):
            return self.model.freight_cost_dc_dc *  (self.model.dist_mat_prod_dc[i,j] + 10.0)/1000.0

        self.model.c = Param(self.model.dcs, self.model.demand_nodes, initialize=cost_init, doc='Transport cost in thousands of dollar per case')
        self.model.c_dc = Param(self.model.dcs, self.model.dcs, initialize=cost_dc, doc = 'transport cost from dc to dc cheaper than other freight cost')
        self.model.c_prod = Param(self.model.production_nodes, self.model.dcs, initialize=cost_prod_dc, doc = 'transport cost from prod to dc cheaper than other freight cost')

        self.model.cases_dc_dem = Var(self.model.dcs, self.model.demand_nodes, self.model.t, bounds=(0.0,None), doc='Shipment quantities in case to demand')
        self.model.cases_held = Var(self.model.dcs, self.model.t, bounds = (0,None), doc = 'cases held at dc')
        self.model.cases_dc_dc = Var(self.model.dcs, self.model.dcs, self.model.t, bounds=(0.0,None), doc = 'shipment quantities in cases between dcs')
        self.model.cases_produced = Var(self.model.production_nodes, self.model.t, bounds = (0.0,None), doc = 'number of units produced')
        self.model.cases_prod_dc = Var(self.model.production_nodes, self.model.dcs, self.model.t, bounds=(0.0,None), doc = 'shipment quantities in cases between production and dcs')

        self.model.continuity_cases = ConstraintList()
        for dc in self.model.dcs:
            for t in self.model.t:
                if t == 0:
                    self.model.continuity_cases.add(self.model.cases_held[dc,t] == start_capacities[dc])
                else:
                    const = self.model.cases_held[dc,t-1]
                    try:
                        const -= sum(self.model.cases_dc_dem[dc, demand_node, t+transit_time_dc_dem[dc,demand_node]] for demand_node in self.model.demand_nodes)
                    except:
                        pass
                    try:
                        const -= sum(self.model.cases_dc_dc[dc,dc2,t+transit_time_dc_dc[dc,dc2]] for dc2 in self.model.dcs)
                    except:
                        pass
                    try:
                        const += sum(self.model.cases_dc_dc[dc1, dc, t+transit_time_dc_dc[dc1,dc]] for dc1 in self.model.dcs)
                    except:
                        pass
                    try:
                        const += sum(self.model.cases_prod_dc[prod_node, dc, t+transit_time_prod_dc[prod_node,dc]] for prod_node in self.model.production_nodes)
                    except:
                        pass

                    self.model.continuity_cases.add(self.model.cases_held[dc,t] == const)

        def supply_rule(model, i,t):
            return sum(model.cases_dc_dem[i,j,t] for j in model.demand_nodes) <= model.cases_held[i,t]
        self.model.supply = Constraint(self.model.dcs, self.model.t, rule=supply_rule, doc='Observe supply limit at plant i')

        def production_rule(model, i, t):
            return model.cases_produced[i,t] <= production_capacity[i]
        self.model.produce = Constraint(self.model.production_nodes, self.model.t, doc='production capacities')

        def production_shipped_rule(model, i, t):
            return model.cases_produced[i, t] == sum(model.cases_prod_dc[i,dc,t] for dc in model.dcs) + 100
        self.model.production_continuity = Constraint(self.model.production_nodes, self.model.t, rule = production_shipped_rule, doc = 'production continuity')

        def demand_rule(model, j,t):
            return sum(model.cases_dc_dem[i,j,t] for i in model.dcs) >= model.demand[j,t] * 0.2
        self.model.demand_constr = Constraint(self.model.demand_nodes, self.model.t, rule=demand_rule, doc='Satisfy demand at market j')

        self.model.demand_costs = sum(self.model.c[i,j]*self.model.cases_dc_dem[i,j,t] for i in self.model.dcs for j in self.model.demand_nodes for t in self.model.t)
        self.model.dc_inter_costs = sum(self.model.c_dc[i,j]*self.model.cases_dc_dc[i,j,t] for i in self.model.dcs for j in self.model.dcs for t in self.model.t)
        self.model.storage_costs = sum(self.model.cases_held[dc,t] * self.model.storage_cost for dc in self.model.dcs for t in self.model.t)
        self.model.prod_trans_costs = sum(self.model.c_prod[i,j]*self.model.cases_prod_dc[i,j,t] for i in self.model.production_nodes for j in self.model.dcs for t in self.model.t)
        self.model.production_costs = sum(self.model.cases_produced[prod,t] * self.model.production_cost for prod in self.model.production_nodes for t in self.model.t)
        self.model.difference_in_demand = sum(sum(self.model.cases_dc_dem[i,j,t] for i in self.model.dcs) +  1*self.model.demand[j,t] for j in self.model.demand_nodes for t in self.model.t)

        def objective_rule(model):
            return model.demand_costs + model.storage_costs + model.dc_inter_costs + model.production_costs + model.prod_trans_costs + 1000000.0*model.difference_in_demand
        self.model.objective = Objective(rule=objective_rule, sense=minimize, doc='Define objective function')
        return self

    def solve(self):
        opt = SolverFactory("cbc", executable='C:\\Users\\Sam Cox\\Documents\\cbc\\Cbc-windeps-win64-msvc15-mtd\\bin\\cbc.exe')
        self.results = opt.solve(self.model)
        return self

if __name__ == '__main__':
    data = CovidData()
    data.get_county_cases()
    data.get_county_coordinates()
    trans_opt = TransOpt(data)
    trans_opt.create_model(trans_opt.data.df_cases)
    trans_opt.solve()
    trans_opt.model.objective.display()
    trans_opt.results.write()
    m= trans_opt.model

    print([[trans_opt.model.cases_produced[i, t].value for i in trans_opt.model.production_nodes] for t in trans_opt.model.t])
    print([[trans_opt.model.cases_held[i, t].value for i in trans_opt.model.dcs] for t in trans_opt.model.t])
    print(trans_opt.model.demand_costs(), trans_opt.model.storage_costs(), trans_opt.model.production_costs(), trans_opt.model.dc_inter_costs(), trans_opt.model.difference_in_demand())
    print([[[m.cases_prod_dc[prod,dc,t].value for prod in trans_opt.model.production_nodes] for dc in m.dcs] for t in m.t])


'''
import the necessary packages
'''
from utils.dataio import get_dataset
from model.AcquisitionLayer import Muestreo
from model.InitializationLayer import BackPropagationLayer, FSI_Initial, FSI_cell
import matplotlib.pyplot as plt
import tensorflow as tf
import numpy as np
import json
import sys
import os

gpus = tf.config.list_physical_devices('GPU')

if len(gpus)> 0:
    print("RUNNING ON GPU")
    for gpu in gpus:
        print("Name:", gpu.name, "  Type:", gpu.device_type)

    try:
        tf.config.set_visible_devices(gpus[0], 'GPU')
    except RuntimeError as e:
    # Visible devices must be set before GPUs have been initialized
        print(e)
else:
    print("running on cpu")

def plot_image(image, path2save):
    plt.imshow(image)
    plt.colorbar()
    plt.savefig(path2save)
    plt.close()

'''
parameters
'''

config_file = sys.argv[-1]
if os.path.exists(config_file):
    print("opening config file")
    print(config_file)
    with open(config_file) as json_file:
        params = json.load(json_file)
        data_params, forward_params, training_params, other_params = params["data"], params["forward_model"], params["training"], params["other"]
else:
    raise Exception("Config file not found: "+ config_file)

# data
SHAPE = data_params["image_shape"]
DATASET_NAME = data_params["dataset_name"]

# Forward model parameters
SNAPSHOTS =  forward_params["number_snapshots"]
TIPO_MUESTREO =  forward_params["tipo_muestreo"]
WAVE_LENGTH =  forward_params["wavelength"]
DX =  forward_params["pixel_size"]
DISTANCE_SENSOR =  forward_params["distance_sensor"]
NORMALIZE_MEASUREMENTS = forward_params["normalize_measurements"]

# Training
BATCH_SIZE = training_params["batch_size"]
RESULTS_FOLDER =  training_params["results_folder"]

if os.path.exists(RESULTS_FOLDER):
    for files in os.listdir(RESULTS_FOLDER):
        os.remove(os.path.join(RESULTS_FOLDER, files))
else:
    os.makedirs(RESULTS_FOLDER)

# other
FLOAT_DTYPE =  "float32"
COMPLEX_DTYPE =  "complex64"
P =  other_params["p"]
K_SIZE = other_params["k_size"]
N_ITER = other_params["n_iterations"]

'''
Data preprocessing
'''

_,  dataset = get_dataset(DATASET_NAME, shape = SHAPE, batch_size = BATCH_SIZE, num_classes = 10, complex_dtype=COMPLEX_DTYPE)
x_image, _ = next(iter(dataset))

'''
Acquisition Layer definition
'''
muestreo_layer = Muestreo(snapshots = SNAPSHOTS, wave_length = WAVE_LENGTH,
                dx = DX,distance = DISTANCE_SENSOR, tipo_muestreo = TIPO_MUESTREO,
                normalize_measurements = NORMALIZE_MEASUREMENTS, codificate = True,
                float_dtype = FLOAT_DTYPE, complex_dtype=COMPLEX_DTYPE, name="Muestreo")

muestras_entrada, mascara_usada, var_de_interes = muestreo_layer(x_image)

'''
image and matrices
'''
x_complex = tf.complex(x_image[0,...,0], x_image[0,...,1])

plot_image(x_image[0,...,0], os.path.join(RESULTS_FOLDER, "imagen_real.png"))
plot_image(x_image[0,...,1], os.path.join(RESULTS_FOLDER, "imagen_imag.png"))
plot_image(tf.math.abs(x_complex), os.path.join(RESULTS_FOLDER, "imagen_abs.png"))
plot_image(tf.math.angle(x_complex), os.path.join(RESULTS_FOLDER, "imagen_angle.png"))

if (TIPO_MUESTREO=="ASM" or TIPO_MUESTREO=="FRESNEL"):
    plot_image(tf.math.abs(var_de_interes[0,0,...]), os.path.join(RESULTS_FOLDER, "T_muestreo_.png"))
    plot_image(tf.math.angle(var_de_interes[0,0,...]), os.path.join(RESULTS_FOLDER, "T_muestreo_.png"))
plot_image(tf.math.abs(mascara_usada[0,0,]), os.path.join(RESULTS_FOLDER, "mascara_abs.png"))
plot_image(tf.math.angle(mascara_usada[0,0,]), os.path.join(RESULTS_FOLDER, "mascara_angle.png"))
#print("mascara rango", tf.reduce_min(tf.math.angle(mascara_usada[0,0,])), tf.reduce_max(tf.math.angle(mascara_usada[0,0,])))
plot_image(muestras_entrada[0,...,0], os.path.join(RESULTS_FOLDER, "muestras_imagen.png"))

'''
run back proopagation
'''

backpropagation_layer  = BackPropagationLayer(tipo_muestreo = TIPO_MUESTREO, complex_dtype = COMPLEX_DTYPE, name = "BackPropagationLayer")
back = backpropagation_layer([muestras_entrada,mascara_usada,var_de_interes])
back_real, back_imag = tf.unstack(back, num=2, axis=-1)
back_complex = tf.complex(back_real[0,], back_imag[0,])

plot_image(back_real[0,], os.path.join(RESULTS_FOLDER, "back_real.png"))
plot_image(back_imag[0,], os.path.join(RESULTS_FOLDER, "back_imag.png"))
plot_image(tf.math.abs(back_complex), os.path.join(RESULTS_FOLDER, "back_abs.png"))
plot_image(tf.math.angle(back_complex), os.path.join(RESULTS_FOLDER, "back_angle.png"))

'''
run fsi initialization
'''
init_initialzation = FSI_Initial(p = P, float_dtype=FLOAT_DTYPE,complex_dtype=COMPLEX_DTYPE, name = "init_initialzation")
Initialation = FSI_cell(p = P, k_size = K_SIZE, tipo_muestreo = TIPO_MUESTREO, float_dtype=FLOAT_DTYPE, complex_dtype=COMPLEX_DTYPE, train_initialization = True, name = "initialization_cell")
Ytr, Z = init_initialzation(muestras_entrada)

for i in range(N_ITER):
    Z = Initialation([Ytr, Z, mascara_usada, var_de_interes])

normalization = tf.reduce_max(tf.math.abs(Z), axis=(1,2,3), keepdims=True)

Z = tf.divide(Z, tf.cast(normalization, tf.complex64))
Z_abs = tf.squeeze(tf.math.abs(Z))
Z_angle = tf.squeeze(tf.math.angle(Z))

kernel_real, kernel_imag = Initialation.get_kernel_initialization()

plot_image(tf.math.real(Z)[0,0], os.path.join(RESULTS_FOLDER, "init_real.png"))
plot_image(tf.math.imag(Z)[0,0], os.path.join(RESULTS_FOLDER, "init_imag.png"))
plot_image(tf.math.abs(Z)[0,0], os.path.join(RESULTS_FOLDER, "init_abs.png"))
plot_image(tf.math.angle(Z)[0,0], os.path.join(RESULTS_FOLDER, "init_angle.png"))
plot_image(kernel_real[...,0,0], os.path.join(RESULTS_FOLDER, "kernel_real.png"))
plot_image(kernel_imag[...,0,0], os.path.join(RESULTS_FOLDER, "kernel_imag.png"))
print("The code ran successfully")


import os

import pytest

from test.fixtures import KafkaFixture, ZookeeperFixture

@pytest.fixture(scope="module")
def version():
    if 'KAFKA_VERSION' not in os.environ:
        return ()
    return tuple(map(int, os.environ['KAFKA_VERSION'].split('.')))

@pytest.fixture(scope="module")
def zookeeper(version, request):
    assert version
    zk = ZookeeperFixture.instance()
    def fin():
        zk.close()
    request.addfinalizer(fin)
    return zk

@pytest.fixture(scope="module")
def kafka_broker(version, zookeeper, request):
    assert version
    k = KafkaFixture.instance(0, zookeeper.host, zookeeper.port,
                              partitions=4)
    def fin():
        k.close()
    request.addfinalizer(fin)
    return k


from pkgutil import iter_modules
from importlib import import_module

def load_configuration(name):
    '''
    加载配置。
    '''

    result = {}
    module = import_module(name)
    for _, child, _ in iter_modules(module.__path__):
        cn = f'{name}.{child}'
        cm = import_module(cn)
        vs = {}
        for k in dir(cm):
            if k[0] != '_':
                vs[k] = getattr(cm, k)
        result[child] = vs
    return result


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
# import umap
# from sklearn.preprocessing import StandardScaler
from sklearn import neighbors
from sklearn import preprocessing

def griz_show(df):
    xedges = np.linspace(0, 1, 100)
    yedges = np.linspace(-0.5, 0.5, 100)

    x = df['psfMag_g'] - df['psfMag_r']
    y = df['psfMag_i'] - df['psfMag_z']

    H, xedges, yedges = np.histogram2d(x, y, bins=(xedges, yedges))
    H = H.T
    plt.imshow(np.log10(H + 1), interpolation='bilinear', origin='low', cmap='jet',
               # norm=LogNorm(vmin=0.1, vmax=10000),
               extent=[xedges[0], xedges[-1], yedges[0], yedges[-1]]
               )

    cls = pd.read_csv(f'{path}class_colors.csv')
    for i in range(cls.__len__()):
        plt.scatter(cls.iloc[i]['g-r'], cls.iloc[i]['i-z'], marker='.', color='black')
        plt.annotate(cls.iloc[i]['class'],
                     xy=(cls.iloc[i]['g-r'] + 0.01, cls.iloc[i]['i-z'] + 0.01))

    plt.show()

def color_combinations(df):
    cls = pd.read_csv(f'{path}class_colors.csv')

    key0 = cls.keys()[0]
    print(key0)
    print(df.__len__())

    df_dist = pd.DataFrame()
    for i in range(cls.__len__()):
        # key = cls.keys()[i]
        dist = np.abs(df['psfMag_g'] - df['psfMag_u'] - cls['g-u'][i]) + \
            np.abs(df['psfMag_g'] - df['psfMag_r'] - cls['g-r'][i]) + \
            np.abs(df['psfMag_g'] - df['psfMag_i'] - cls['g-i'][i]) + \
            np.abs(df['psfMag_i'] - df['psfMag_z'] - cls['i-z'][i])
        df_dist[cls['class'][i]] = dist

    min = pd.DataFrame(df_dist.idxmin(axis=1), columns=['class'])
    df['class'] = df_dist.idxmin(axis=1)
    print(min['class'].value_counts())
    # print(df_dist)
    # print(min)
    min['color'] = min['class'].apply(ord)-65

    # print(min)
    # min['color'] = ord
    # print(cls['class'][min])
    df['u-g'] = df['psfMag_u'] - df['psfMag_g']
    df['u-r'] = df['psfMag_u'] - df['psfMag_r']
    df['u-i'] = df['psfMag_u'] - df['psfMag_i']
    df['u-z'] = df['psfMag_u'] - df['psfMag_z']

    df['g-r'] = df['psfMag_g'] - df['psfMag_r']
    df['g-i'] = df['psfMag_g'] - df['psfMag_i']
    df['g-z'] = df['psfMag_g'] - df['psfMag_z']

    df['r-i'] = df['psfMag_r'] - df['psfMag_i']
    df['r-z'] = df['psfMag_r'] - df['psfMag_z']

    df['i-z'] = df['psfMag_i'] - df['psfMag_z']

    import matplotlib.cm as cm

    # colors = cm.rainbow(np.linspace(0, 1, len(ys)))
    # plt.scatter(df['g-r'], df['i-z'], marker='.', c=min['color'],
    #             cmap='Set3', alpha=0.5, edgecolors='none')

    colors_values = list(set(min['color']))
    colors_names = cls['class'].to_list()
    colors = dict(zip(colors_names, colors_values))
    # print(colors_values, colors_names)
    # for area in [100, 300, 500]:
    #     plt.scatter([], [], c='k', alpha=0.3, s=area,
    #                 label=str(area) + ' km$^2$')
    # plt.legend(scatterpoints=1, frameon=False, labelspacing=1, title='City Area')

    # cmap = plt.cm.Greys
    # cmin = np.min(colors_values)
    # import matplotlib.colors
    # norm = matplotlib.colors.Normalize(vmin=np.min(colors_values), vmax=np.max(colors_values))
    # for i, val in enumerate(colors_values):
    #     plt.scatter([], [], c=cmap(colors_values[i]), alpha=1,
    #                 label=colors_names[i], norm=norm)
    # plt.legend(scatterpoints=1, frameon=False, labelspacing=1)

    # plt.legend(colors_values, colors_names)
    # plt.legend()

    # for i in range(cls.__len__()):
    #     plt.scatter(cls.iloc[i]['g-r'], cls.iloc[i]['i-z'], marker='.', color='black')
    #     plt.annotate(cls.iloc[i]['class'],
    #                  xy=(cls.iloc[i]['g-r'] + 0.01, cls.iloc[i]['i-z'] + 0.01), weight='bold')
    #     sns.jointplot(x="x", y="y", data=df, kind="kde");

    # N = 11
    # for i in range(N):
    #     sns.palplot(sns.light_palette(sns.color_palette()[0], i+1))
    # print(len(sns.color_palette()))
    # plt.show()
    # for x in df['class'].map(d_cls):
    #     print(x)
    # c = [sns.color_palette()[x] for x in df['class'].map(d_cls)]
    # print(c)
    # plt.show()

    sns.pairplot(df[['u-g', 'u-r', 'u-i', 'u-z',
                     'g-r', 'g-i', 'g-z',
                     'r-i', 'r-z',
                     'i-z', 'class']], hue='class')
    plt.show()

def show_umap(df):
    reducer = umap.UMAP(n_neighbors=50,
                        min_dist=0.01,
                        n_components=4,
                        metric='euclidean')

    new_data = df[['u-g', 'u-r', 'u-i', 'u-z',
                   'g-r', 'g-i', 'g-z',
                   'r-i', 'r-z',
                   'i-z']].values
    scaled_new_data = StandardScaler().fit_transform(new_data)
    embedding = reducer.fit_transform(scaled_new_data)
    print(embedding.shape)

    # for x in df['class'].map(d_cls):
    #     print(x, end=',')
    print(set(df['class']))
    labels = list(set(df['class']))

    a = sns.palplot(sns.color_palette('Spectral', 11))
    plt.scatter(
        embedding[:, 0],
        embedding[:, 1],
        c=[sns.color_palette('Paired')[x] for x in df['class'].map(d_cls)],
    marker='.')
    plt.gca().set_aspect('equal', 'datalim')
    plt.title('UMAP projection of the Penguin dataset', fontsize=24)
    umap.plot.points(reducer, labels=labels)
    plt.show()

    # fig, ax = plt.subplots(1, figsize=(14, 10))
    # plt.scatter(*embedding.T, s=0.1, c=target, cmap='Spectral', alpha=1.0)
    # plt.setp(ax, xticks=[], yticks=[])
    # cbar = plt.colorbar(boundaries=np.arange(11) - 0.5)
    # cbar.set_ticks(np.arange(10))
    # cbar.set_ticklabels(classes)
    # plt.show()

def load_class_from_spectra():
    df = pd.read_csv(f'{path}sdss_from_spectra.csv')
    print(df)
    # print(df[df['complex'].isna() == True])
    # print(df['complex'].value_counts())
    df = df.dropna(subset=['complex'])
    print(df['complex'].value_counts())

    # print(set(df['name']))
    benoit_names = set(df['name'])
    print('asteroids spectra:', len(benoit_names))
    spec = df.groupby('name', as_index=False).first()
    # print(spec)

    sdss = pd.read_csv(f'{path}sso_tot4c.csv')
    cond = sdss['bcolor_ru'] & sdss['bcolor_rg'] & sdss['bcolor_ri'] & sdss['bcolor_rz'] & \
           sdss['bastrom_u'] & sdss['bastrom_g'] & sdss['bastrom_r'] & sdss['bastrom_i'] & \
           sdss['bastrom_z'] & sdss['bphot_u'] & sdss['bphot_g'] & sdss['bphot_r'] & \
           sdss['bphot_i'] & sdss['bphot_z']

    sdss = sdss[cond]
    coincidences = sdss[sdss['Name'].isin(benoit_names)]

    # sdss_names = list(set(coincidences['Name']))
    # print(sdss_names)
    # print(coincidences['Name'])
    print('asteroids sdss:', len(set(coincidences['Name'])))

    classes = np.zeros(coincidences.__len__(), dtype=(np.str))
    for i in range(coincidences.__len__()):
        obj = coincidences.iloc[i]['Name']
        val = spec.loc[spec['name'] == obj]['complex'].values[0]
        classes[i] = val
    coincidences.loc[:, 'complex'] = classes
    # print(coincidences)

    dictarr = []
    for key in d_cls:
        # dictarr.append(d_cls[key])
        dictarr.append(key)

    d_stat = pd.DataFrame(d_cls.items(), columns=['complex', 'id'])
    del d_stat['id']
    # cond = coincidences.loc['class'] == d_stat.loc['class']
    # d_stat.loc['total'] = pd.Series(coincidences[cond].sum())

    coincidences.loc[:, 'u-g'] = coincidences.loc[:, 'psfMag_u'] - coincidences.loc[:, 'psfMag_g']
    coincidences.loc[:, 'u-r'] = coincidences.loc[:, 'psfMag_u'] - coincidences.loc[:, 'psfMag_r']
    coincidences.loc[:, 'u-i'] = coincidences.loc[:, 'psfMag_u'] - coincidences.loc[:, 'psfMag_i']
    coincidences.loc[:, 'u-z'] = coincidences.loc[:, 'psfMag_u'] - coincidences.loc[:, 'psfMag_z']
    coincidences.loc[:, 'g-r'] = coincidences.loc[:, 'psfMag_g'] - coincidences.loc[:, 'psfMag_r']
    coincidences.loc[:, 'g-i'] = coincidences.loc[:, 'psfMag_g'] - coincidences.loc[:, 'psfMag_i']
    coincidences.loc[:, 'g-z'] = coincidences.loc[:, 'psfMag_g'] - coincidences.loc[:, 'psfMag_z']
    coincidences.loc[:, 'r-i'] = coincidences.loc[:, 'psfMag_r'] - coincidences.loc[:, 'psfMag_i']
    coincidences.loc[:, 'r-z'] = coincidences.loc[:, 'psfMag_r'] - coincidences.loc[:, 'psfMag_z']
    coincidences.loc[:, 'i-z'] = coincidences.loc[:, 'psfMag_i'] - coincidences.loc[:, 'psfMag_z']

    ast_group = coincidences.groupby('complex')
    print(ast_group['complex'].count())
    d_stat = d_stat.join(ast_group['complex'].count(), on='complex', rsuffix='_r')
    d_stat = d_stat.join(ast_group['u-g'].mean(), on='complex', rsuffix='_mean')
    d_stat = d_stat.join(ast_group['u-r'].mean(), on='complex', rsuffix='_mean')
    d_stat = d_stat.join(ast_group['u-i'].mean(), on='complex', rsuffix='_mean')
    d_stat = d_stat.join(ast_group['u-z'].mean(), on='complex', rsuffix='_mean')
    d_stat = d_stat.join(ast_group['g-r'].mean(), on='complex', rsuffix='_mean')
    d_stat = d_stat.join(ast_group['g-i'].mean(), on='complex', rsuffix='_mean')
    d_stat = d_stat.join(ast_group['g-z'].mean(), on='complex', rsuffix='_mean')
    d_stat = d_stat.join(ast_group['r-i'].mean(), on='complex', rsuffix='_mean')
    d_stat = d_stat.join(ast_group['r-z'].mean(), on='complex', rsuffix='_mean')
    d_stat = d_stat.join(ast_group['i-z'].mean(), on='complex', rsuffix='_mean')

    d_stat = d_stat.join(ast_group['u-g'].std(), on='complex', rsuffix='_std')
    d_stat = d_stat.join(ast_group['u-r'].std(), on='complex', rsuffix='_std')
    d_stat = d_stat.join(ast_group['u-i'].std(), on='complex', rsuffix='_std')
    d_stat = d_stat.join(ast_group['u-z'].std(), on='complex', rsuffix='_std')
    d_stat = d_stat.join(ast_group['g-r'].std(), on='complex', rsuffix='_std')
    d_stat = d_stat.join(ast_group['g-i'].std(), on='complex', rsuffix='_std')
    d_stat = d_stat.join(ast_group['g-z'].std(), on='complex', rsuffix='_std')
    d_stat = d_stat.join(ast_group['r-i'].std(), on='complex', rsuffix='_std')
    d_stat = d_stat.join(ast_group['r-z'].std(), on='complex', rsuffix='_std')
    d_stat = d_stat.join(ast_group['i-z'].std(), on='complex', rsuffix='_std')

    # d_stat = d_stat.join(ast_group['u-g'].median(), on='complex', lsuffix='median_')

    # d_stat = d_stat.rename({'u-g_r': ''})
    # d_stat = d_stat.join(ast_group['u-r'].mean(), on='complex', lsuffix='mean_')
    # d_stat = d_stat.join(ast_group['u-r'].median(), on='complex', lsuffix='median_')
    d_stat = d_stat.rename(columns={'complex_r': 'total', 'u-g': 'mean_u-g', 'u-r': 'mean_u-r'})

    pd.set_option("display.precision", 3)
    print(d_stat)
    # # d_stat = pd.merge(d_stat, ast_group['complex'].count(), left_on='complex', right_on='complex')
    # # print(d_stat)
    #
    # tmp = np.zeros(d_stat.__len__())
    # for i in range(d_stat.__len__()):
    #     cond = coincidences['complex'] == d_stat.iloc[i]['complex']
    #     tmp[i] = len(coincidences[cond])
    # d_stat['total'] = tmp
    # print(d_stat)

    print(coincidences['complex'].value_counts())
    isshow = False
    if isshow:
        fig, (ax1, ax2) = plt.subplots(1, 2, gridspec_kw={'width_ratios': [10, 1]})
        pal = sns.color_palette('Paired', 11)
        cond_c = (coincidences['complex'] == 'A') | (coincidences['complex'] == 'K') | \
                 (coincidences['complex'] == 'L') | (coincidences['complex'] == 'Q') | \
                 (coincidences['complex'] == 'B') | (coincidences['complex'] == 'D')
        cond_c = ~cond_c
        x = coincidences['psfMag_g'][cond_c] - coincidences['psfMag_r'][cond_c]
        y = coincidences['psfMag_i'][cond_c] - coincidences['psfMag_z'][cond_c]
        ax1.scatter(x=x, y=y,
                    c=[pal[x] for x in coincidences['complex'][cond_c].map(d_cls)],
                    marker='.')
        ax2 = sns.barplot(y=dictarr, x=np.ones(len(dictarr)), palette=pal)
        ax2.set_xticks([])
        ax2.spines['right'].set_visible(False)
        ax2.spines['left'].set_visible(False)
        ax2.spines['top'].set_visible(False)
        ax2.spines['bottom'].set_visible(False)

        ax1.set_xlabel('g-r')
        ax1.set_ylabel('i-z')
        ax1.set_xlim([0.1, 1.0])
        ax1.set_ylim([-0.7, 0.7])

        # plt.savefig('./figs/spec_class1.png', dpi=300)
        plt.show()

    issave = True
    if issave:
        coincidences.to_csv('.\data\ccoincidences.csv', index=False)

    return coincidences

def knn_class():
    from sklearn.model_selection import train_test_split
    from sklearn.model_selection import cross_val_score, cross_val_predict

    coincidences = pd.read_csv('.\data\ccoincidences.csv')
    X = coincidences[color_list]
    y = coincidences['complex']

    my_scaler = preprocessing.StandardScaler()
    my_scaler.fit(X)
    X_2 = my_scaler.transform(X)

    a_tot = coincidences['complex'].value_counts()

    for state in range(0, 1):
        X_train, X_test, y_train, y_test = \
            train_test_split(X_2, y, stratify=y, random_state=state)

        all_scores = []
        n_neighb = np.arange(10, 13)
        for n_neighbors in n_neighb:
            print(n_neighbors)
            knn_clf = neighbors.KNeighborsClassifier(n_neighbors=n_neighbors)
            knn_clf.fit(X_train, y_train)
            # knn_clf.score(X_train, y_train)
            # print(knn_clf.score(X_train, y_train))
            # knn_scores = cross_val_score(knn_clf, X_train, y_train, cv=3)

            Z = cross_val_predict(knn_clf, X, y, cv=3)
            cond = (Z != y)

            all_scores.append((n_neighbors,
                               # knn_scores.mean(),
                               # knn_scores.std(),
                               coincidences.loc[:, ['complex']][cond].__len__()
                               ))
            wrong = coincidences.loc[:, ['complex']][cond]
            a1 = wrong['complex'].value_counts()
            print((a_tot - a1)/ a_tot)
            # print(coincidences['complex'].value_counts())

        # res = np.array(sorted(all_scores, key=lambda x: x[1], reverse=True))
        all_scores = np.array(all_scores)
        # for item in res:
        #     print(item)
        # Z = knn_clf.predict(X)
        # cond = (Z != y)
        # df_wrong = coincidences.loc[:, ['complex']][cond]
        # print(df_wrong.__len__())

        # my_scaler = preprocessing.StandardScaler()
        # my_scaler.fit(X)
        # X_2 = my_scaler.transform(X)
        # # X_2 = preprocessing.scale(X)
        # print(X_2.std(axis=0))
        #
        # neighb = np.arange(2, 50)
        # # print(neighb)
        # score1 = np.zeros(neighb.shape[0])
        # for i, n_neighbors in enumerate(neighb):
        #     # we create an instance of Neighbours Classifier and fit the data.
        #     clf = neighbors.KNeighborsClassifier(n_neighbors, weights='uniform',n_jobs=4)
        #     clf.fit(X, y)
        #     Z = clf.predict(X)
        #     cond = (Z != y)
        #     df_wrong = coincidences.loc[:, ['complex']][cond]
        #     score1[i] = df_wrong.__len__()
        #     print(i, end=', ')
        #
        # print()
        # score2 = np.zeros(neighb.shape[0])
        # for i, n_neighbors in enumerate(neighb):
        #     # we create an instance of Neighbours Classifier and fit the data.
        #     clf = neighbors.KNeighborsClassifier(n_neighbors, weights='uniform', n_jobs=4)
        #     clf.fit(X_2, y)
        #     Z = clf.predict(X_2)
        #     cond = (Z != y)
        #     df_wrong = coincidences.loc[:, ['complex']][cond]
        #     score2[i] = df_wrong.__len__()
        #     print(i, end=', ')
        #
        isshow = True
        if isshow:
            plt.ylabel('wrong calss')
            plt.xlabel('neighbors')

            plt.plot(n_neighb, all_scores[:, 1], label='X')
            # plt.plot(n_neighb, all_scores[:, 1], label='X')
            # plt.errorbar(x=n_neighb, y=all_scores[:, 1], yerr=all_scores[:, 2])

    plt.legend()
    plt.show()
        #
    # # cond = (Z != y)
    # # print(coincidences.loc[:, ['complex']][cond])
    # # print(test_score)

if __name__ == '__main__':
    path = f'./data/'
    d_cls = {"A": 0, "B": 1, "C": 2, "D": 3, "K": 4, "L": 5, "Q": 6, "S": 7, "U": 8, "V": 9, "X": 10}
    color_list = ['u-g', 'u-r', 'u-i', 'u-z',
                  'g-r', 'g-i', 'g-z',
                  'r-i', 'r-z',
                  'i-z']
    # print(d_cls.items())

    # df = pd.read_csv(f'{path}last/sso_tot4c.csv')
    # print('Load complete.')
    # cond = df['bcolor_ru'] & df['bcolor_rg'] & df['bcolor_ri'] & df['bcolor_rz'] & \
    #     df['bastrom_u'] & df['bastrom_g'] & df['bastrom_r'] & df['bastrom_i'] & df['bastrom_z']
    # df = df[['psfMag_u', 'psfMag_g', 'psfMag_r', 'psfMag_i', 'psfMag_z']][cond].reset_index()
    # griz_show()
    # color_combinations()
    # coinc = load_class_from_spectra()
    knn_class()


import os
import sys
import platform
import numpy as np
import matplotlib.pyplot as mp

def init_chart():
    mp.gcf().set_facecolor(np.ones(3) * 240 / 255)
    mp.title('Polynomial Function', fontsize=20)
    mp.xlabel('x', fontsize=14)
    mp.ylabel('y', fontsize=14)
    mp.tick_params(which='both', top=True, right=True,
                   labelright=True, labelsize=10)
    mp.grid(linestyle=':')

def draw_chart(x, y):
    mp.plot(x, y, c='orangered', label='$y=x^3+2x^2+3x+4$')
    mp.legend()

def show_chart():
    mng = mp.get_current_fig_manager()
    if 'Windows' in platform.system():
        mng.window.state('zoomed')
    else:
        mng.resize(*mng.window.maxsize())
    mp.show()

def main(argc, argv, envp):
    x = np.linspace(-10, 10, 201)
    f = np.poly1d([1., 2., 3., 4.])
    y = f(x)
    init_chart()
    draw_chart(x, y)
    show_chart()
    return 0

if __name__ == '__main__':
    exit(main(len(sys.argv), sys.argv, os.environ))


import torch
import torch.nn as nn
import torch.nn.functional as F

class InceptionNet(nn.Module):
    def __init__(self, aux_logits=True, num_classes=2):
        super(InceptionNet, self).__init__()
        self.aux_logits = aux_logits
        self.conv1 = ConvBlock(
            in_channels=3,
            out_chanels=64,
            kernel_size=(7, 7),
            stride=(2, 2),
            padding=(3, 3),
        )
        self.conv2 = ConvBlock(64, 192, kernel_size=3, stride=1, padding=1)
        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
        self.inception3a = InceptionBlock(192, 64, 96, 128, 16, 32, 32)
        self.inception3b = InceptionBlock(256, 128, 128, 192, 32, 96, 64)
        self.inception4a = InceptionBlock(480, 192, 96, 208, 16, 48, 64)
        self.inception4b = InceptionBlock(512, 160, 112, 224, 24, 64, 64)
        self.inception4c = InceptionBlock(512, 128, 128, 256, 24, 64, 64)
        self.inception4d = InceptionBlock(512, 112, 144, 288, 32, 64, 64)
        self.inception4e = InceptionBlock(528, 256, 160, 320, 32, 128, 128)
        self.inception5a = InceptionBlock(832, 256, 160, 320, 32, 128, 128)
        self.inception5b = InceptionBlock(832, 384, 192, 384, 48, 128, 128)
        self.avgpool = nn.AvgPool2d(kernel_size=7, stride=1)
        self.dropout = nn.Dropout(p=0.4)
        self.fc = nn.Linear(1024, num_classes)

        if self.aux_logits:
            self.aux1 = InceptionAux(512, num_classes)
            self.aux2 = InceptionAux(528, num_classes)
        else:
            self.aux1 = self.aux2 = None

    def forward(self, x):
        x = self.conv1(x)
        x = self.maxpool(x)
        x = self.conv2(x)
        x = self.maxpool(x)
        x = self.inception3a(x)
        x = self.inception3b(x)
        x = self.maxpool(x)
        x = self.inception4a(x)

        if self.aux_logits and self.training:
            aux1 = self.aux1(x)

        x = self.inception4b(x)
        x = self.inception4c(x)
        x = self.inception4d(x)

        if self.aux_logits and self.training:
            aux2 = self.aux2(x)

        x = self.inception4e(x)
        x = self.maxpool(x)
        x = self.inception5a(x)
        x = self.inception5b(x)
        x = self.avgpool(x)
        x = x.reshape(x.shape[0], -1)
        x = self.dropout(x)
        x = self.fc(x)

        if self.aux_logits and self.training:
            return aux1, aux2, x
        return x

class ConvBlock(nn.Module):
    def __init__(self, in_channels, out_chanels, **kwargs):
        super(ConvBlock, self).__init__()
        self.conv = nn.Conv2d(in_channels, out_chanels, **kwargs)
        self.bn = nn.BatchNorm2d(out_chanels)

    def forward(self, x):
        return F.relu(self.bn(self.conv(x)))

class InceptionBlock(nn.Module):
    def __init__(
        self,
        in_channels,
        out_1x1,
        red_3x3,
        out_3x3,
        red_5x5,
        out_5x5,
        out_pool,
    ):
        super(InceptionBlock, self).__init__()
        self.branch1 = ConvBlock(in_channels, out_1x1, kernel_size=1)
        self.branch2 = nn.Sequential(
            ConvBlock(in_channels, red_3x3, kernel_size=1, padding=0),
            ConvBlock(red_3x3, out_3x3, kernel_size=3, padding=1),
        )
        self.branch3 = nn.Sequential(
            ConvBlock(in_channels, red_5x5, kernel_size=1),
            ConvBlock(red_5x5, out_5x5, kernel_size=5, padding=2),
        )
        self.branch4 = nn.Sequential(
            nn.MaxPool2d(kernel_size=3, padding=1, stride=1),
            ConvBlock(in_channels, out_pool, kernel_size=1),
        )

    def forward(self, x):
        branches = (self.branch1, self.branch2, self.branch3, self.branch4)
        return torch.cat([branch(x) for branch in branches], 1)

class InceptionAux(nn.Module):
    def __init__(self, in_channels, num_classes):
        super(InceptionAux, self).__init__()
        self.dropout = nn.Dropout(p=0.7)
        self.pool = nn.AvgPool2d(kernel_size=5, stride=3)
        self.conv = ConvBlock(in_channels, 128, kernel_size=1)
        self.fc1 = nn.Linear(2048, 1024)
        self.fc2 = nn.Linear(1024, num_classes)

    def forward(self, x):
        x = self.pool(x)
        x = self.conv(x)
        x = x.reshape(x.shape[0], -1)
        x = F.relu(self.fc1(x))
        x = self.dropout(x)
        x = self.fc2(x)
        return x


import addonHandler
addonHandler.initTranslation()

import globalPluginHandler
import tones
import config
import ui
import api
import globalVars
import gui
from scriptHandler import script, getLastScriptRepeatCount

configName = 'objLocSound'
scriptCategory = _(u'物件位置音效')
if configName not in config.conf:
	config.conf[configName] = {'switch': True}

conf = config.conf[configName]
if conf['switch'] == u'False':
	conf['switch'] = False

class GlobalPlugin(globalPluginHandler.GlobalPlugin):
	def __init__(self):
		super(GlobalPlugin, self).__init__()
		self.currentObject = None
		self.currentObjectLocation = None
		# 持續觀察物件位置，用於動態物件，例如跑馬燈，或是某些沒有事件的焦點移動，比如瀏覽模式。
		self.timer = gui.NonReEntrantTimer(self.playLocationSound)
		if self.switch:
			self.start()

	def terminate(self):
		if hasattr(self, 'timer'):
			self.stop()

	def _get_switch(self):
		return conf.get('switch')

	def _set_switch(self, value):
		conf['switch'] = value

	def start(self):
		self.timer.Start(10)

	def stop(self):
		self.timer.Stop()

	@script(_(u'提示開關'), scriptCategory, 'KB:NVDA+l')
	def script_toggleSwitch(self, gesture):
		switch = not self.switch
		self.switch = switch
		methods = (self.stop, self.start)
		methods[switch]()
		messages = (_(u'關閉物件位置提示音效'), _(u'開啟物件位置提示音效'))
		ui.message(messages[switch])

	def playLocationSound(self):
		# 播放當前瀏覽物件中心點所在位置音效
		try:
			obj = api.getNavigatorObject()
			location = obj.location
			if location is None or self.currentObject == obj and location == self.currentObjectLocation:
				return

			self.currentObject = obj
			self.currentObjectLocation = location
			x, y = self.getCenterCoordinate(obj)

			self.playCoordinateSound(x, y)
		except TypeError:
			pass

	def getCenterCoordinate(self, obj):
		l, t, w, h = obj.location
		x = l + (w // 2)
		y = t + (h // 2)
		return (x, y)

	def playCoordinateSound(self, x, y):
		screenWidth, screenHeight = api.getDesktopObject().location[2:]
		if 0 <= x <= screenWidth and 0 <= y <= screenHeight:
			minPitch = config.conf['mouse']['audioCoordinates_minPitch']
			maxPitch = config.conf['mouse']['audioCoordinates_maxPitch']
			curPitch = minPitch + ((maxPitch - minPitch) * ((screenHeight - y) / float(screenHeight)))
			brightness = config.conf['mouse']['audioCoordinates_maxVolume']
			leftVolume = int((85 * ((screenWidth - float(x)) / screenWidth)) * brightness)
			rightVolume = int((85 * (float(x) / screenWidth)) * brightness)
			tones.beep(curPitch, 40, left=leftVolume, right=rightVolume)

	@script(u'播放當前物件左上與右下角座標音效，連按兩次報讀詳細資訊', scriptCategory, 'KB:NVDA+a')
	def script_playAreaSound(self, gesture):
		obj = api.getNavigatorObject()
		l, t, w, h = obj.location
		if getLastScriptRepeatCount():
			message = _(u'左上角：') + u' X:' + str(l) + u', Y:' + str(t) + u'\n'
			message += _(u'寬：') + u' ' + str(w) + u', ' + _(u'高：') + u' ' + str(h)
			return ui.message(message)

		self.playCoordinateSound(l, t)
		gui.wx.CallLater(100, self.playCoordinateSound, l+w, t+h).Start()


import unittest
from datasets import Impl_DatasetsWindow
from unittest.mock import MagicMock
from PyQt5.QtWidgets import QMessageBox

class TestHelpWindow(unittest.TestCase):

    def setUp(self):
        self.window = Impl_DatasetsWindow()

    def test_help_button_click(self):
        # simulate the click event on the Help button
        self.window.btn_Help.click()

        # ensure that the QMessageBox was created and shown with the correct information
        mbox = QMessageBox()
        mbox.setIcon(QMessageBox.Information)
        mbox.setWindowTitle("Help")
        mbox.setText("Here is all the information about the DataSet Window")
        mbox.setInformativeText("Click the Datasets button, press the Load Dataset on the top left corner and select the csv or xml file that contains the dataset.(It seems to work mostly with the XMLs with the CVS you have to manually pick the “preset” on the right).The tool will recognize the type of report and preload the recommended features for best prediction accuracy.\nYou can add or remove columns and the start labelling the dataset which also can be saved to used later.")
        mbox.setStandardButtons(QMessageBox.Ok | QMessageBox.Cancel)
        mbox.exec_ = MagicMock()
        mbox.exec_()
        mbox.exec_.assert_called_once()


import os, random
import pickle
from typing import Optional
import torch
from torch.utils.data import Dataset, DataLoader
import pytorch_lightning as pl

class SeqAttnData(Dataset):
    def __init__(self, seq: dict):
        self.seq = seq
        self.idx2caseid = {
            i: case
            for i, case in enumerate(self.seq)
        }

    def __len__(self):
        return len(self.idx2caseid)

    def __getitem__(self, idx):
        case = self.idx2caseid[idx]
        data = self.seq[case]
        return case, data['L']['gesture'], data['R']['gesture'], data['label']

    @staticmethod
    def collate_fn(batch):
        # max seq len
        max_seq_len = 882
        max_len_L = max(
            len(L) for _, L, _, _ in batch
        )
        max_len_L = min(max_len_L, max_seq_len)
        input_ids_L = torch.zeros((len(batch), max_len_L))
        attn_mask_L = torch.zeros((len(batch), max_len_L))
        max_len_R = max(
            len(R) for _, _, R, _ in batch
        )
        max_len_R = min(max_len_R, max_seq_len)
        input_ids_R = torch.zeros((len(batch), max_len_R))
        attn_mask_R = torch.zeros((len(batch), max_len_R))
        labels = torch.zeros((len(batch),))
        cases = []
        for i, (case, L, R, label) in enumerate(batch):
            # shift right and add 3 (0,1,2 preserved)
            # 0: CLS
            # 1: SEP
            # 2: PAD
            L = (L + 3)[:(max_len_L - 2)].tolist()
            L_ids = (
                    [0] +
                    L +
                    [1] +
                    [2] * (max_len_L - 2 - len(L))
            )
            L_mask = [1] * (len(L) + 2) + [0] * (max_len_L - 2 - len(L))
            input_ids_L[i] = torch.tensor(L_ids)
            attn_mask_L[i] = torch.tensor(L_mask)

            R = (R + 3)[:(max_len_R - 2)].tolist()
            R_ids = (
                    [0] +
                    R +
                    [1] +
                    [2] * (max_len_R - 2 - len(R))
            )
            R_mask = [1] * (len(R) + 2) + [0] * (max_len_R - 2 - len(R))
            input_ids_R[i] = torch.tensor(R_ids)
            attn_mask_R[i] = torch.tensor(R_mask)

            labels[i] = label
            cases.append(case)

        return {
            "input_ids_L": input_ids_L.long(),
            "attn_mask_L": attn_mask_L.bool(),
            "input_ids_R": input_ids_R.long(),
            "attn_mask_R": attn_mask_R.bool(),
            "label": labels,
            "case": cases
        }

class MCDataModule(pl.LightningDataModule):
    def __init__(self,
        data_dir: str,
        batch_size: int, num_workers: int,
        k_fold: int,
        **kwargs
    ):
        super().__init__()
        self.batch_size = batch_size
        self.num_workers = num_workers
        self.data_dir = data_dir
        self.dataset = {}
        self.k_fold = k_fold

    def setup(self, stage: Optional[str] = None):
        seq = {}
        for split in ['train', 'valid', 'test']:
            # file = os.path.join(self.data_dir, f"k0", f"{split}_processed.pkl")
            file = os.path.join(self.data_dir, f"k0", f"{split}_combined.pkl")
            assert os.path.exists(file)

            with open(file, "rb") as f:
                seq = {**seq, **pickle.load(f)}

        keys = list(seq.keys())
        random.seed(self.k_fold)
        random.shuffle(keys)
        if "con" in self.data_dir:
            # drop 3 cases
            if "Gronau" in self.data_dir:
                assert len(seq) == 77
                traink, testk = keys[:62], keys[62:]
            else:
                assert len(seq) == 48
                traink, testk = keys[:41], keys[41:]
        elif "Gronau" in self.data_dir:
            assert len(seq) == 80
            traink, testk = keys[:64], keys[64:]
        else:
            assert len(seq) == 50
            traink, testk = keys[:40], keys[40:]
        train, test = {}, {}
        for k, v in seq.items():
            if k in traink:
                train = {**train, k:v}
            else:
                test = {**test, k:v}
        self.dataset["train"] = SeqAttnData(train)
        self.dataset["valid"] = SeqAttnData(test)

    def get_loader(self, split) -> DataLoader:
        return DataLoader(
            self.dataset[split], batch_size=self.batch_size,
            collate_fn=SeqAttnData.collate_fn, drop_last=False,
            num_workers=self.num_workers, pin_memory=True,
            shuffle=(split=="train")
        )

    def train_dataloader(self) -> DataLoader:
        return self.get_loader("train")

    def val_dataloader(self) -> DataLoader:
        return self.get_loader("valid")


from typing import Dict
from models.flow import Flow
import warnings
from subprocess import Popen, PIPE

import numpy as np
import pyshark.tshark.tshark
import pyshark
from cryptography import x509
from cryptography.hazmat.backends import default_backend

class PcapReader(object):
    """Reader object for extracting features from .pcap files
    """

    def __init__(self, verbose=False):
        """Reader object for extracting features from .pcap files

        Args:
        --------
            verbose (bool, optional): Boolean indicating whether to be verbose in reading
        """
        # Set verbosity level
        self.verbose = verbose

    def read(self, path: str) -> dict:
        """Read TCP and UDP packets from .pcap file given by path.
            Automatically choses fastest available backend to use.

        Args:
        --------
            path (str): Path to .pcap file to read.

        Raises:
        -------
            ModuleNotFoundError: If TShark is not installed
            OSError: If there is problem reading file (usually caused because not being able to allocate memory)
        Returns:
        -------
            dict: Dictionary of flow_key -> Flow()
            Where features consist of:

                0) Filename of capture
                1) Protocol TCP/UDP
                2) TCP/UDP stream identifier
                3) Timestamp of packet
                4) Length of packet
                5) IP packet source
                6) IP packet destination
                7) TCP/UDP packet source port
                8) TCP/UDP packet destination port
                9) SSL/TLS certificate if exists, else None
        """
        # print(time.time(), ":\tstart reading pcap")
        # If verbose, print which file is currently being read
        if self.verbose:
            print("Reading {}...".format(path))
        # print(time.time(), ":\tend reading pcap")
        # Check if we can use fast tshark read or slow pyshark read
        packets = self.read_pyshark(path)
        return self.combine(packets)

    def read_tshark(self, path: str) -> np.ndarray:
        """Read TCP and UDP packets from file given by path using tshark backend
            Automatically choses fastest available backend to use.

        Args:
        --------
            path (str): Path to .pcap file to read.

        Returns:
        -------
            np.ndarray: Array of shape=(n_packets, n_features)
            Where features consist of:

                0) Filename of capture
                1) Protocol TCP/UDP
                2) TCP/UDP stream identifier
                3) Timestamp of packet
                4) Length of packet
                5) IP packet source
                6) IP packet destination
                7) TCP/UDP packet source port
                8) TCP/UDP packet destination port
                9) SSL/TLS certificate if exists, else None
        """
        # Create Tshark command
        command = ["tshark", "-r", path, "-Tfields",
                   "-e", "frame.time_epoch",
                   "-e", "tcp.stream",
                   "-e", "udp.stream",
                   "-e", "ip.proto",
                   "-e", "ip.src",
                   "-e", "tcp.srcport",
                   "-e", "udp.srcport",
                   "-e", "ip.dst",
                   "-e", "tcp.dstport",
                   "-e", "udp.dstport",
                   "-e", "ip.len"]
        ##"-e", "ssl.handshake.certificate"
        # Initialise result
        result = list()

        # Call Tshark on packets
        process = Popen(command, stdout=PIPE, stderr=PIPE)
        # Get output
        out, err = process.communicate()

        # Give warning message if any
        if err:
            warnings.warn("Error reading file: '{}'".format(
                err.decode('utf-8')))

        # Read each packet
        for packet in filter(None, out.decode('utf-8').split('\n')):
            # Get all data from packets
            packet = packet.split()

            # Perform check on packets
            if len(packet) < 8: continue

            # Perform check on multiple ip addresses
            packet[3] = packet[3].split(',')[0]
            packet[5] = packet[5].split(',')[0]
            packet[7] = packet[7].replace(',', '')

            # Parse certificate
            if len(packet) > 8:
                # Get first certificate
                cert = packet[8].split(',')[0]
                # Transform to hex
                cert = bytes.fromhex(cert.replace(':', ''))
                # Read as certificate
                cert = x509.load_der_x509_certificate(cert, default_backend())
                # Set packet as serial number
                packet[8] = cert.serial_number
            else:
                packet.append(None)

            # Add packet to result
            result.append([path] + packet)

        # Get result as numpy array
        result = np.asarray(result)

        # Check if any items exist
        if not result.shape[0]:
            return np.zeros((0, 8), dtype=object)

        # Change protocol number to text
        protocols = {'17': 'udp', '6': 'tcp'}
        result[:, 3] = [protocols.get(x, 'unknown') for x in result[:, 3]]

        # Return in original order
        return result[:, [0, 3, 2, 1, 8, 4, 6, 5, 7, 9]]

    def read_pyshark(self, path: str) -> np.ndarray:
        """Read TCP and UDP packets from file given by path
            using pyshark backend

        Args:
        --------
            path (str): Path to .pcap file to read.

        Returns:
        -------
            np.ndarray: Array of shape=(n_packets, n_features)
            Where features consist of:

                0) Filename of capture
                1) Protocol TCP/UDP
                2) TCP/UDP stream identifier
                3) Timestamp of packet
                4) Length of packet
                5) IP packet source
                6) IP packet destination
                7) TCP/UDP packet source port
                8) TCP/UDP packet destination port
                9) SSL/TLS certificate if exists, else None
        """
        # If verbose, print which file is currently being read
        if self.verbose:
            counter_a = 0
            counter_b = 0

        # Read pcap file
        pcap = iter(pyshark.FileCapture(path))

        # Initialise result
        result = list()

        # Loop over packets
        while True:
            try:
                packet = next(pcap)
            except ModuleNotFoundError as ex:
                raise ex
            except OSError as ex:
                raise ex
            except Exception as ex:
                warnings.warn("Pyshark error: '{}'".format(ex))
                break

            if not ("TCP" in packet or "UDP" in packet):
                continue

            try:

                d = []
                # Get required packet data
                if packet.layers[1].stream:
                    d = [path,
                        packet.layers[1].layer_name,  # Get
                        packet.layers[1].stream,  # Get stream ID
                        packet.sniff_timestamp,  # Get packet timestamp
                        packet.length,  # Get packet length
                        packet.layers[0].src,  # Get source IP or IPv6 (fixed)
                        packet.layers[0].dst,  # Get destination IP or IPv6 (fixed)
                        packet.layers[1].srcport,  # Get source port
                        packet.layers[1].dstport,  # Get destination port
                        None]
                elif packet.layers[2].stream:
                    d = [path,
                        packet.layers[2].layer_name,  # Get
                        packet.layers[2].stream,  # Get stream ID
                        packet.sniff_timestamp,  # Get packet timestamp
                        packet.length,  # Get packet length
                        packet.layers[1].src,  # Get source IP or IPv6 (fixed)
                        packet.layers[1].dst,  # Get destination IP or IPv6 (fixed)
                        packet.layers[2].srcport,  # Get source port
                        packet.layers[2].dstport,  # Get destination port
                        None]
                else:
                    continue

                # Check whether SSL/TLS certificate is in packet
                if "SSL" in packet and \
                        packet.ssl.get("handshake_certificate") is not None:
                    # Get certificate
                    cert = packet.ssl.get('handshake_certificate')
                    # Parse cert to bytes
                    cert = bytes.fromhex(cert.replace(':', ''))
                    # Parse x509 certificate as DER
                    cert = x509.load_der_x509_certificate(cert,
                                                        default_backend())
                    # Get serial number - TODO extend with other features?
                    d[-1] = cert.serial_number

                # Append data item to result
                result.append(d)
            except Exception as ex:
                warnings.warn(f"Cannot read pcap: {path}")

        # Close capture
        pcap.close()

        # Return result as numpy array
        return np.array(result)

    def combine(self, packets: np.ndarray) -> dict:
        """Combine individual packets into a flow representation

            Parameters
            ----------
            packets : np.array of shape=(n_samples_packets, n_features_packets)
                Output from Reader.read

            Returns
            -------
            flows : dict
                Dictionary of flow_key -> Flow()
            """
        # Initialise result
        result = dict()

        # For each packet, add it to a flow
        for packet in packets:
            key = (packet[0], packet[1], packet[2])
            # Add packet to flow
            result[key] = result.get(key, Flow()).add(packet)

        # Return result
        return result


#INFO: Preliminary TOF support
#TODO: Add different camera models
#TODO: Write the full evd data (noise vector, angles)
#TODO: Add the grayscale image values (of the render image)
#TODO: Add the foldback effect of ToF Cameras.
#TODO: Add output of depthmap
#TODO: Simulate the sampling (i.e. PMD outputs only 1500 different depth values)
#      See SR4000 user manual for details
#TODO: Implement radial distortion
#INFO: SR4000
#      176x144 pixel
#      Standard FOV: 43.6°(h) 34.6°(v)    Focal Length: 10mm
#      Wide FOV: 69°(h) 56°(v)            Focal Length: 5.8mm
#         pixel_width (mm): 0.0453
#         pixel_height(mm): 0.0430
#      Model 1: unambiguity distance 5m
#      Absolute accuracy of central pixels +-10mm
#      Repeatability of central pixels (sigma) 4mm
#
#      Model 2: unambiguity distance 10m
#      Absolute accuracy of central pixels +-15mm
#      Repeatability of central pixels (sigma) 6mm

import math
import sys
import os
import struct
import ctypes
import time
import random
import bpy
import numpy
import blensor.globals
import blensor.scan_interface
from blensor import evd
from blensor import mesh_utils

from mathutils import Vector, Euler, Matrix

def deg2rad(deg):
    return deg*math.pi/180.0

def rad2deg(rad):
    return rad*180.0/math.pi

def tuples_to_list(tuples):
    l = []
    for t in tuples:
        l.extend(t)
    return l

parameters = {"max_dist":20,"noise_mu":0.0,"noise_sigma":0.004, "backfolding":False, "xres": 176, "yres": 144, "lens_angle_w": 43.6, "lens_angle_h": 34.6, "flength": 10.0}

def addProperties(cType):
    global parameters
    cType.tof_max_dist = bpy.props.FloatProperty( name = "Scan distance", default = parameters["max_dist"], min = 0, max = 1000, description = "How far the laser can see" )
    cType.tof_noise_mu = bpy.props.FloatProperty( name = "Noise mu", default = parameters["noise_mu"], description = "The center of the gaussian noise" )
    cType.tof_noise_sigma = bpy.props.FloatProperty( name = "Noise sigma", default = parameters["noise_sigma"], description = "The sigma of the gaussian noise" )
    cType.tof_backfolding = bpy.props.BoolProperty( name = "Backfolding", default = parameters["backfolding"], description = "Should backfolding be simulated" )
    cType.tof_xres = bpy.props.IntProperty( name = "X resolution", default = parameters["xres"], description = "Horizontal resolution" )
    cType.tof_yres = bpy.props.IntProperty( name = "Y resolution", default = parameters["yres"], description = "Vertical resolution" )
    cType.tof_lens_angle_w = bpy.props.FloatProperty( name = "Horizontal FOV", default = parameters["lens_angle_w"], description = "Horizontal opening angle" )
    cType.tof_lens_angle_h = bpy.props.FloatProperty( name = "Vertical FOV", default = parameters["lens_angle_h"], description = "Vertical opening angle" )
    cType.tof_focal_length = bpy.props.FloatProperty( name = "Focal length", default = parameters["flength"], description = "Focal length" )

def scan_advanced(scanner_object, max_distance = 10.0, evd_file=None, add_blender_mesh = False,
                  add_noisy_blender_mesh = False, tof_res_x = 176, tof_res_y = 144,
                  lens_angle_w=43.6, lens_angle_h=34.6, flength = 10.0,  evd_last_scan=True,
                  noise_mu=0.0, noise_sigma=0.004, timestamp = 0.0, backfolding=False,
                  world_transformation=Matrix()):

    inv_scan_x = scanner_object.inv_scan_x
    inv_scan_y = scanner_object.inv_scan_y
    inv_scan_z = scanner_object.inv_scan_z

    start_time = time.time()

    #10.0mm is currently the distance between the focal point and the sensor
    sensor_width = 2 * math.tan(deg2rad(lens_angle_w/2.0)) * 10.0
    sensor_height = 2 * math.tan(deg2rad(lens_angle_h/2.0)) * 10.0

    if tof_res_x == 0 or tof_res_y == 0:
        raise ValueError("Resolution must be > 0")

    pixel_width = sensor_width / float(tof_res_x)
    pixel_height = sensor_height / float(tof_res_y)

    bpy.context.scene.render.resolution_percentage

    width = bpy.context.scene.render.resolution_x
    height = bpy.context.scene.render.resolution_y
    cx = float(tof_res_x) /2.0
    cy = float(tof_res_y) /2.0

    evd_buffer = []
    rays = []
    ray_info = []

    ray = Vector([0.0,0.0,0.0])
    for x in range(tof_res_x):
        for y in range(tof_res_y):
            """Calculate a vector that originates at the principal point
               and points to the pixel in the sensor. This vector is then
               scaled to the maximum scanning distance
            """
            physical_x = float(x-cx) * pixel_width
            physical_y = float(y-cy) * pixel_height
            physical_z = -float(flength)
            ray.xyz = [physical_x, physical_y, physical_z]
            ray.normalize()
            final_ray = max_distance*ray
            rays.extend([final_ray[0],final_ray[1],final_ray[2]])

            """ pitch and yaw are added for completeness, normally they are
                not provided by a ToF Camera but can be derived
                from the pixel position and the camera parameters.
            """
            yaw = math.atan(physical_x/flength)
            pitch = math.atan(physical_y/flength)

            ray_info.append([yaw, pitch, timestamp])

    returns = blensor.scan_interface.scan_rays(rays, max_distance, inv_scan_x = inv_scan_x, inv_scan_y = inv_scan_y, inv_scan_z = inv_scan_z)

    evd_storage = evd.evd_file(evd_file, tof_res_x, tof_res_y, max_distance)

    reusable_vector = Vector([0.0,0.0,0.0,0.0])
    for i in range(len(returns)):
        idx = returns[i][-1]
        distance_noise =  random.gauss(noise_mu, noise_sigma)
        #If everything works substitute the previous line with this
        #distance_noise =  pixel_noise[returns[idx][-1]] + random.gauss(noise_mu, noise_sigma)

        reusable_vector.xyzw = [returns[i][1],returns[i][2],returns[i][3],1.0]
        vt = (world_transformation * reusable_vector).xyz
        v = [returns[i][1],returns[i][2],returns[i][3]]
        vector_length = math.sqrt(v[0]**2+v[1]**2+v[2]**2)
        norm_vector = [v[0]/vector_length, v[1]/vector_length, v[2]/vector_length]

        vector_length_noise = vector_length+distance_noise
        if backfolding:
           #Distances > max_distance/2..max_distance are mapped to 0..max_distance/2
           if vector_length_noise >= max_distance/2.0:
               vector_length_noise = vector_length_noise - max_distance/2.0

        reusable_vector.xyzw = [norm_vector[0]*vector_length_noise, norm_vector[1]*vector_length_noise, norm_vector[2]*vector_length_noise,1.0]
        v_noise = (world_transformation * reusable_vector).xyz

        evd_storage.addEntry(timestamp = ray_info[idx][2], yaw =(ray_info[idx][0]+math.pi)%(2*math.pi), pitch=ray_info[idx][1], distance=vector_length, distance_noise=vector_length_noise, x=vt[0], y=vt[1], z=vt[2], x_noise=v_noise[0], y_noise=v_noise[1], z_noise=v_noise[2], object_id=returns[i][4], color=returns[i][5], idx=returns[i][-1])

    if evd_file:
        evd_storage.appendEvdFile()

    if not evd_storage.isEmpty():
        scan_data = numpy.array(evd_storage.buffer)
        additional_data = None
        if scanner_object.store_data_in_mesh:
            additional_data = evd_storage.buffer

        if add_blender_mesh:
            mesh_utils.add_mesh_from_points_tf(scan_data[:,5:8], "Scan", world_transformation, buffer=additional_data)

        if add_noisy_blender_mesh:
            mesh_utils.add_mesh_from_points_tf(scan_data[:,8:11], "NoisyScan", world_transformation, buffer=additional_data)

        bpy.context.scene.update()

    end_time = time.time()
    scan_time = end_time-start_time
    print ("Elapsed time: %.3f"%(scan_time))

    return True, 0.0, scan_time

# This Function creates scans over a range of frames

def scan_range(scanne_object, frame_start, frame_end, filename="/tmp/tof.evd", frame_time = (1.0/24.0), fps = 24, add_blender_mesh=False, add_noisy_blender_mesh=False, max_distance = 20.0, last_frame = True, noise_mu = 0.0, noise_sigma = 0.0, backfolding=False, tof_res_x = 176, tof_res_y=144, lens_angle_w=43.6, lens_angle_h=34.6, flength = 10.0, world_transformation=Matrix()):

    start_time = time.time()

    time_per_frame = 1.0 / float(fps)

    try:
        for i in range(frame_start,frame_end):

            bpy.context.scene.frame_current = i

            ok,start_radians,scan_time = scan_advanced(scanner_object=scanner_object,
                    evd_file = filename ,
                    add_blender_mesh=add_blender_mesh,
                    add_noisy_blender_mesh=add_noisy_blender_mesh,
                    max_distance=max_distance,
                    timestamp = float(i) * frame_time,
                    noise_mu = noise_mu, noise_sigma=noise_sigma, backfolding=backfolding,
                    tof_res_x = tof_res_x, tof_res_y = tof_res_y,
                    lens_angle_w = lens_angle_w, lens_angle_h = lens_angle_h, flength = flength,
                    world_transformation=world_transformation)

            if not ok:
                break
    except:
        print ("Scan aborted")

    if last_frame:
        evd_file = open(filename,"a")
        evd_file.buffer.write(struct.pack("i",-1))
        evd_file.close()

    end_time = time.time()
    print ("Total scan time: %.2f"%(end_time-start_time))

######################################################