# -*- coding: utf-8 -*-
"""
Created on Fri Jul 07 10:10:39 2017

@author: 160229
"""

import json
import logging

from helper import close, get_slots
from runbook import getRunbookById
from runlog import Runlog, save
from slack import getSlackUserById

headers = {'content-type': 'application/json'}
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)

""" --- Functions that control the bot's behavior --- """
def respond(intent_request):
    """
    fulfill assign intent
    """
    output_session_attributes = intent_request['sessionAttributes'] if intent_request['sessionAttributes'] is not None else {}
    current_user = getSlackUserById(intent_request['userId'])
    runbook_id = get_slots(intent_request)["RunbookId"]
    assignee = get_slots(intent_request)["SlackUser"]

    runbook=getRunbookById(runbook_id)
    runlog_entry = Runlog.create(runbook_id, runbook['name'], assignee, current_user)
    save(runlog_entry)
    print('respond save runlog={}'.format(str(runlog_entry)))
    output_session_attributes['runlog']=json.dumps(runlog_entry.__dict__)

    print('respond output_session_attributes={}, slots={}'.format(str(output_session_attributes), str(get_slots(intent_request))))
    return close(output_session_attributes, 'Fulfilled', {'contentType': 'PlainText', 'content': 'Assigned \''+runbook_id+'\' to '+assignee})

""" --- Intents --- """
def dispatch(intent_request):
    print('dispatch userId={}, intentName={}'.format(intent_request['userId'], intent_request['currentIntent']['name']))
    return respond(intent_request)

""" --- Main handler --- """
def lambda_handler(event, context):
    logger.debug('event.bot.name={}'.format(event['bot']['name']))
    return dispatch(event)


# -*- coding: utf-8 -*-
"""
Created on Sun Jul 25 15:05:25 2021

@author: anton
"""

import pandas as pd
import numpy as np
import datetime
from sklearn.feature_extraction.text import CountVectorizer
##reading in the df
base_df = pd.read_csv(r'ready_to_go_1020_220726.csv')

#cut date to YYmmDD
base_df.date = [date for date in base_df.date.dt.date]

#converting date dtype to date
base_df.date = base_df.date.astype('datetime64[ns]')

# get month and date range
m_range = pd.date_range(start=min(base_df.date), end=max(base_df.date),freq='M')
d_range = pd.date_range(start=min(base_df.date), end=max(base_df.date),freq='D')
#date organizing logic

def term_timeline(index,df):
    # creating an empty dataframe
    ts_term_df = pd.DataFrame()
    #for date in day_index
    for date in index:
        #date filter/new df for each day
        daily_df = df[df['date'] == date]
         #implement an if statement to skip days with no data
        if len(daily_df) > 0:

            #count vectorizer instance
            cv = CountVectorizer(stop_words = 'english',ngram_range=(1,2))
            terms = cv.fit_transform(daily_df.tweet)
            vec = np.transpose(terms.toarray())
            f_names = cv.get_feature_names()
            vec_df = pd.DataFrame(vec,index=f_names)
            #new_df: index, date col, term col, freq col

            #creating a frequency dict
            frequency_dict = {}

            for i in range(0,len(vec_df)-1):
                frequency_dict[vec_df.index[i]] = sum(vec_df.iloc[i,])

            #converting f dict to df
            dterms = pd.DataFrame.from_dict(frequency_dict, orient = 'index').reset_index()
            #adding date for each df
            dterms['date'] = date

            #changin column names

            dterms = dterms.rename(columns={'index':'terms', 0:'t_freq'})

            #sorting and isolating top terms
            dterms = dterms.sort_values(by = ['t_freq'], ascending = False).head(20)

            #controling for first/next df
            if len(ts_term_df) == 0:
                 ts_term_df = dterms
            else:
                ts_term_df = ts_term_df.append(dterms)
    #0:t_freq renaming only works outside of the function for some reason

    return ts_term_df

# monthy tweet counts
from collections import Counter
#using Counter function to aggregate duplicate month as each entry == tweet
tweet_counts = Counter(base_df.date.dt.month)

ts_df = term_timeline(d_range,base_df)
## use month range to group days by, aggregate all token sums? Can Tableu can be used?
ts_df.to_csv('ts_df_1020_220726.csv')


import os
import subprocess
from contextlib import contextmanager
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Iterator, NamedTuple, Set

class PackageLabInfo(NamedTuple):
    venv_location: Path
    package_location: str
    package_name: str
    package_version: str
    all_package_files: Set[str]

def create_linux_venv(dir: Path, venv_location: Path, all_package_files: Path, package: str) -> int:
    command = f"""#!/bin/bash
    python3 -m venv {venv_location}
    source {venv_location / 'bin' / 'activate'}
    pip3 install --upgrade wheel pip
    pip3 install --no-cache-dir {package}
    pip3 show {package} --files > {all_package_files}
    """

    install_file = Path(dir) / "install.sh"
    with open(install_file, "w") as f:
        f.write(command)

    process = subprocess.run(f"bash {install_file}", shell=True)
    return process.returncode

def create_windows_venv(dir: Path, venv_location: Path, all_package_files: Path, package: str) -> int:
    command = f"""#!/usr/bin/pwsh
    python -m venv {venv_location}
    . {venv_location / 'Scripts' / 'Activate.ps1'}
    pip install --user --upgrade wheel pip
    pip install --no-cache-dir {package}
    pip show {package} --files | Out-File -FilePath {all_package_files}
    """

    install_file = Path(dir) / "install.ps1"
    with open(install_file, "w") as f:
        f.write(command)

    process = subprocess.run(f"powershell {install_file}", shell=True)
    return process.returncode

@contextmanager
def create_venv_and_install(package: str) -> Iterator[PackageLabInfo]:
    with TemporaryDirectory() as dir:
        dir = Path(dir)
        venv_location = dir / "venv"
        all_package_files = Path(dir) / "pkg-files.txt"

        if os.name == "nt":
            returncode = create_windows_venv(dir, venv_location, all_package_files, package)
        else:
            returncode = create_linux_venv(dir, venv_location, all_package_files, package)

        if returncode != 0:
            return None

        with open(all_package_files) as f:
            in_files_section = False
            all_files = set()
            for line in f.readlines():
                line = line.replace("\0", "")  # Remove null chars. For Windows compatibility.
                if line.startswith("Version: "):
                    pkg_version = line[len("Version: ") :].strip()
                elif line.startswith("Location: "):
                    pkg_location = line[len("Location: ") :].strip()
                elif line.startswith("Files:"):
                    in_files_section = True
                elif in_files_section:
                    all_files.add(line.strip())
        yield PackageLabInfo(venv_location, pkg_location, package, pkg_version, all_files)


import matplotlib.pyplot as plt
import numpy as np
import csv
from loaddata import *
import matplotlib.cm as cm
from matplotlib import colors

WIDTH_NUM = 20
HEIGHT_NUM = 20

TEST_NUM = 1000
TEST_TARGET = list(range(5010, 5030))
NUM_SIDES = list(range(3, 6))
BUFFER_OPT = "buffer_001"
RESULT_FILE = "problem2/model3/01_" + BUFFER_OPT + ".txt"
RESULT_FILE_PATH = "../result/" + RESULT_FILE

WHOLE_RANGE = [-5.0, 5., -5., 5.]

test_x_data = []
test_y_data = []

tf.reset_default_graph()

for numSides in NUM_SIDES:
    print(str(numSides) + " =========== LOAD DATA ===========")
    _test_x_data, _test_y_data = load_raster_data("../data/problem2/raster_pc/" + BUFFER_OPT + "/test_" + str(numSides) + ".csv")
    test_x_data.extend(_test_x_data)
    test_y_data.extend(_test_y_data)

# result file
print("Accuraccy : " + open(RESULT_FILE_PATH, 'r').readline())
result_data = np.loadtxt(RESULT_FILE_PATH, dtype=np.float32, skiprows=1)
# result_data = result_data.reshape(result_data.shape[0], 1)

# polygon
for test_i in TEST_TARGET:
    coords_x = []
    coords_y = []

    # create discrete colormap
    cmap = colors.ListedColormap(['red', 'blue'])
    bounds = [0, 10, 20]
    norm = colors.BoundaryNorm(bounds, cmap.N)

    fig, ax = plt.subplots()
    ax.imshow(data, cmap=cmap, norm=norm)

    # draw gridlines
    ax.grid(which='major', axis='both', linestyle='-', color='k', linewidth=2)
    ax.set_xticks(np.arange(WHOLE_RANGE[0], WHOLE_RANGE[1], WIDTH_NUM));
    ax.set_yticks(np.arange(WHOLE_RANGE[2], WHOLE_RANGE[3], HEIGHT_NUM));

    plt.show()


import threading
import itertools
import time

# this will later be used to add a loading '...' to some text
done_dot=False

def dotdotdot(text):
    for c in itertools.cycle(['.', '..', '...','']):
        if done_dot:
            break
        sys.stdout.write('\r'+text+c)
        sys.stdout.flush()
        time.sleep(0.3)
    sys.stdout.write('\nDone!')

# needed for the next function
import subprocess
import importlib

# function that imports a library if it is installed, else installs it and then imports it
def getpack(package):
    try:
        return importlib.import_module(package)
        # import package
    except ImportError:
        subprocess.call([sys.executable, "-m", "pip", "install", package],
                        stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
        return importlib.import_module(package)
        # import package

# import necessary packages
from eff_fron import *
pd = getpack("pandas")
yf = getpack("yfinance")
bs = getpack("bs4")
requests = getpack("requests")
plt = getpack("matplotlib.pyplot")
import random

# get the tickers of all stocks in the SP500
def sp500_tickers():
    resp = requests.get('http://en.wikipedia.org/wiki/List_of_S%26P_500_companies')
    soup = bs.BeautifulSoup(resp.text, 'lxml')
    table = soup.find('table', {'class': 'wikitable sortable'})
    tickers = []
    for row in table.findAll('tr')[1:]:
        ticker = row.findAll('td')[0].text
        ticker = ticker.replace("\n", "")
        tickers.append(ticker)

    return tickers

# get the returns for a given list of tickers
def get_returns(tickers):
    returns = []

    # keep track of number of requests
    n_requests = 0
    # for tickers with insufficient return data
    insufficient = []
    print("Downloading return-information, this might takes some time.", end='\r')
    for i in tickers:
        # get data
        data_tick = yf.Ticker(i)
        # daily changes of past 1-years in percent
        ret = data_tick.history(period="1y").Close
        ret = [ret1 / ret2 - 1 for ret1, ret2 in zip(ret[1:], ret)]
        if len(ret) == 250:
            returns.append(ret)
        else:
            insufficient.append(i)
        del ret

        n_requests += 1
        print(f"Downloading return-information, this might takes some time. ({n_requests}/{len(tickers)})", end='\r')

    print("\nReturns downloaded.")
    if len(insufficient) > 0:
        print(f"The following tickers will be omitted because of insufficient data: {', '.join(insufficient)}.\n"
              f"This could be due to the company going public within the last year or no information being available on"
              f" Yahoo finance.")
    print("\n")

    return returns

# generate random portfolio weights for the random portfolios in the visualization
def random_weights(n, k):
    return np.random.dirichlet(np.ones(n), size=k)

def main():
    # for dotdotdot function
    global done_dot

    print("Welcome, this is a cmd-line interface tool to calculate minimum risk portfolio weights for a given set of "
          "stocks and a selected expected return  (daily).")
    print("In other words, this script calculated the portfolio weights and standard deviation for a portfolio at point"
          " mu_p of the efficient frontier.")
    print("If you are unfamiliar with Moskowitz's Efficient Frontier please read the README.md file.\n")

    while True:
        done_dot = False
        sel = input("\nIf you want to run a simulation for a custom group of (U.S.) stocks enter '0'.\n"
                    "If you want to run a simulation for all stocks within the SP500 "
                    "enter '1'. Disclaimer: depending on your hardware this might take very long or not work at all.\n"
                    "If you want to run a simulation for a random sample of stocks within the SP500 enter '2'\n\n"
                    "To end the program enter 'q'\n")

        # end program/loop if q is entered
        if sel == 'q':
            break

        # if custom portfolio get ticker input
        elif sel == '0':
            tickers = [x for x in input("Please enter the tickers of the stocks you want to include in your portfolio."
                                        " Seperate them by a space: ").split()]

        # if not custom either draw a sample of SP500 or use all (depending on selection)
        elif sel == '1' or sel == '2':
            tickers = sp500_tickers()
            if sel == '2':
                tickers = random.sample(tickers, int(input("How many securities should be in your random sample: ")))

        else:
            print("invalid selection")

        # get returns and whether short-selling should be incorporated
        returns = get_returns(tickers)
        option = int(input("If you want to incorporate short selling enter '1' else enter '0': "))

        # expected returns and covariance matrix
        mus = [np.mean(i) for i in returns]
        cov_m = np.cov(returns)

        # load weights result into df
        results = pd.DataFrame(list(zip(tickers, mus, [np.std(i) for i in returns])),
                               columns=["Ticker", "expected return", "standard dev"])
        pd.set_option('display.max_rows', results.shape[0] + 1)
        print(results)

        # get target return
        mu_target = float(input(f"Please enter your target return (e.g.: 5% as 0.05):"))

        # prepare a loading message
        t = threading.Thread(target=dotdotdot, args=("Calculating optimal portfolio, this may take some time",))
        # starting loading... thread
        t.start()

        # get risk-free rate e.g. 3-month t-bill
        # rf = yf.Ticker("^IRX").history(period="1y").Close[-1] / 100
        # rf = (1 + rf) ** (1 / 60) - 1
        rf = 0.0

        # risk minimizing portfolio for target return (or max possible return)
        res = optimal_portfolio(mus, cov_m, mu_target, rf, option)

        # minimum variance portfolio
        min_var = min_var_portfolio(mus, cov_m, rf)

        # calculate some portfolios on the frontier for visualization
        r_range = np.linspace(min_var.mu, max(mus), 75)
        efficient_fron = efficient_frontier(mus, cov_m, rf, r_range)
        del r_range

        # The code below could be used to calculate the tangency portfolio. Since this programm works with daily
        # returns, and I could not find a satisfactory solution for a daily risk free rate, it is not implemented.
        # tang_port = tangency_portfolio(mus, cov_m,rf)

        # end loading '...'
        done_dot = True
        time.sleep(0.3)
        print("\n")

        # display results and additional information
        print(f"The risk-minimizing portfolio for your selection has a standard deviation of {round(res.std,5)}, "
              f"an expected return of {round(res.mu,5)} and the following weights on the securities:")

        # make sure all rows will be displayed

        results["weights"] = [round(i, 5) for i in res.weights]
        pd.set_option('display.max_rows', results.shape[0] + 1)
        print(results)

        # if target mu and real mu differ more than 1 percent
        if abs(mu_target - res.mu) >= mu_target*0.01:
            print(f"\nYour target return of {mu_target} could not be reached. Try to integrate shortselling.")

        if min_var.mu > res.mu:
            print(f"\nAttention: though this portfolio minimizes the risk for your chosen return, the minimal risk "
                  f"portfolio achieves a higher mean return ({round(min_var.mu,5)}) with a standard deviation of "
                  f"{round(min_var.std,5)}. The weights on the minimal risk portfolio are: {min_var.weights}\n")

        # visualization option
        if 'y' in input("Would you like to see this result on the efficient frontier? (y/n)   "):
            # random portfolios for scatter plot
            rand_mus = []
            rand_std = []
            rand_sr = []

            for i in random_weights(len(tickers), len(tickers)*4000):
                r_mu = sum([i[j] * mus[j] for j in range(len(tickers))])
                rand_mus.append(r_mu)
                r_sig = sigma_p(i, cov_m)
                rand_std.append(r_sig)
                rand_sr.append((r_mu-rf)/r_sig)

            plt.scatter(rand_std, rand_mus, c=rand_sr, cmap='YlGnBu', marker='o', s=10, alpha=0.2)
            plt.title('Portfolio Optimization based on Efficient Frontier')
            plt.xlabel('Standard deviation')
            plt.ylabel('Expected return')
            plt.colorbar().set_label('Sharpe ratio')

            # chosen portfolio, minimum variance portfolio, tangency portfolio and efficient frontier
            plt.scatter(res.std, res.mu, marker='*', s=25, c='red', label="Your portfolio")
            plt.scatter(min_var.std, min_var.mu, marker='D', s=25, c='darkgreen', label="Minimum variance")
            # plt.scatter(tang_port.std, tang_port.mu, marker='^', s=25, c='gold', label="Tangency portfolio")
            plt.plot([p.std for p in efficient_fron], [p.mu for p in efficient_fron], linestyle='-.', color='black',
                     label='efficient frontier')

            plt.legend()

            plt.show()

            del rand_mus, rand_std

        if 'n' in input("Do you want to run another simulation? (y/n)   "):
            break

    print("Thank you.\nGoodbye.")

main()


# utils.py

import re
import pandas as pd
from datetime import datetime
from typing import List, Union

def export_to_excel(list_dfs: List[pd.DataFrame], directory_in: str) -> None:
    """
    Export data to an Excel file.
    :param list_dfs: List of DataFrames to export.
    :param directory_in: The directory path for saving the Excel file.
    """
    match = re.search(r"\\([^\\]+)\\[^\\]*\.pdf", directory_in)
    name_excel = match.group(1).replace(" ", "_") if match else "default_name"

    df = pd.concat(list_dfs)
    df.reset_index(drop=True, inplace=True)
    df.to_excel(directory_in[:-6] +
                datetime.today().strftime("%d-%m-%Y") + "_" + name_excel + ".xlsx", sheet_name="Sheet 1")

def search_string(df: pd.DataFrame, match_words: List[str]) -> pd.DataFrame:
    """
    Search for a series of words in the DataFrame and return the matching rows.
    :param df: The DataFrame to search in.
    :param match_words: List of words to search for.
    :return: DataFrame containing matching rows.
    """
    pattern = re.compile("|".join(match_words))
    return df.loc[df[0].str.contains(pattern, na=False)]

def to_float(data: Union[float, str]) -> float:
    """
    Convert a string to a floating-point number.
    :param data: The input data to convert.
    :return: The converted floating-point number.
    """
    data = str(data)
    if data:
        return float(("-" + data.replace("-", "") if "-" in data else data)
                     .replace(".", "")
                     .replace(",", ".")
                     .replace("$", ""))
    return 0

def even_odd_lists(n: int) -> List[List[int]]:
    """
    Generate lists of even and odd numbers up to a given number (n).
    :param n: The maximum number in the lists.
    :return: A list containing two sublists - one for even numbers and one for odd numbers.
    """
    even = [num for num in range(2, n + 1, 2)]
    odd = [num for num in range(1, n + 1, 2)]
    return [even, odd]

def complete_columns(df: pd.DataFrame) -> pd.DataFrame:
    """
    Complete a DataFrame with empty columns until it has a total of 4 columns.
    :param df: The DataFrame to complete.
    :return: The DataFrame with empty columns added.
    """
    while len(df.columns) != 4:
        df[len(df.columns)] = ""
    return df


from django import forms
from django.core.exceptions import ValidationError
from django.core.validators import RegexValidator

class UserPhoneForm(forms.Form):

    phone = forms.CharField(

        label='Введите пожалуйста Ваш номер',

        min_length='17',

        max_length='17',

        required=True,

        widget=forms.TextInput(

            attrs={
                'type': 'tel',
                'name': 'phone',
                'id': 'user_phone',
                'class': 'form-control',
                'placeholder': '+7(999)-999-99-99',
                'autocomplete': 'off',
                'pattern': '\+[7][(]\d{3}[)]-\d{3}-\d{2}-\d{2}',
            },

        ),

        error_messages={

            'required': 'Пожалуйста введите актуальный номер',

            'min_length': 'Номер должен содержать 17 цифр'
        },
    )

    def clean_phone(self):

        data = self.cleaned_data.get('phone')

        if len(data) < 17:
            raise ValidationError(
                _('Номер телефона должен содержать не менее 17 символов'),
                code='invalid length',)
        return data


from django.conf.urls import url
from . import views

urlpatterns = [
    url(r'^$', views.index, name='index'),
    url(r'^cities/$', views.cities, name='cities'),
    url(r'city/(?P<pk>\d+)', views.citydetail, name='city-detail'),
    url(r'gig/(?P<pk>\d+)', views.gigdetail, name='gig-detail'),
    url(r'^gigs/$', views.gigs, name='gigs'),
    url(r'pub/(?P<pk>\d+)', views.pubdetail, name='pub-detail'),
    url(r'fastfood/(?P<pk>\d+)', views.fastfooddetail, name='fastfood-detail'),
    url(r'hospital/(?P<pk>\d+)', views.hospitaldetail, name='hospital-detail'),
    url(r'strip/(?P<pk>\d+)', views.stripdetail, name='strip-detail'),
    url(r'^entertainments/$', views.entertainments, name='entertainments'),
    url(r'^recorddeals/$', views.recorddeals, name='recorddeals'),
    url(r'^recordstudios/$', views.recordstudios, name='recordstudios'),
    url(r'^videoclip/$', views.videoclip, name='videoclip'),
    url(r'^recorddeals/(?P<pk>\d+)', views.recorddealsdetail, name='recorddeals-detail'),
    url(r'^recordstudios/(?P<pk>\d+)', views.recordstudiosdetail, name='recordstudios-detail'),
    url(r'^videoclip/(?P<pk>\d+)', views.videoclipdetail, name='videoclip-detail'),

]


import pandas as pd
import matplotlib.pyplot as plt
from pandas.plotting import register_matplotlib_converters
import matplotlib.dates as mdates

register_matplotlib_converters()

excl = pd.read_csv('./datasets/EXCL.JK.csv', index_col=False, parse_dates=['Date'])
fren = pd.read_csv('./datasets/FREN.JK.csv', index_col=False, parse_dates=['Date'])
isat = pd.read_csv('./datasets/ISAT.JK.csv', index_col=False, parse_dates=['Date'])
tlkm = pd.read_csv('./datasets/TLKM.JK.csv', index_col=False, parse_dates=['Date'])

# print(excl.head())

plt.style.use('seaborn-darkgrid')
plt.figure(figsize=(12,8))
ax = plt.subplot()
plt.plot(excl['Date'], excl['Close'])
plt.plot(fren['Date'], fren['Close'])
plt.plot(isat['Date'], isat['Close'])
plt.plot(tlkm['Date'], tlkm['Close'])
ax.set_title('Harga Historis Saham Provider Telco Indonesia')

ax.set_xlabel('Tanggal')
ax.set_ylabel('Rupiah (IDR)')
legends = ['PT XL Axiata Tbk', 'PT Smartfren Telecom Tbk', 'PT Indosat Tbk', 'PT Telekomunikasi Indonesia Tbk']
plt.legend(legends, bbox_to_anchor=(0., 1.0, 1., .102), loc=9, ncol=4, mode="expand", borderaxespad=0.)
ticks = pd.date_range(start='20190315', end='20190616', freq='W', name='point')
myFmt = mdates.DateFormatter('%d-%m-%Y')
ax.xaxis.set_major_formatter(myFmt)
plt.xticks(ticks, rotation=35, fontsize=9)
plt.show()


"""In order to plug in nicely to the command line wrapper, element
submodules should expose functions called ``main()`` and
``get_parser()``.

The ``get_parser()`` function should look like this::

  def get_parser(parser=None):
    if parser is None:
      parser = argparse.ArgumentParser()
    # element-specific args:
    parser.add_argument('obs_id', help="The obs_id to analyze.")
    parser.add_arugment('--config-file', help="Config file.")
    return parser

When called by the ``so-site-pipeline`` wrapper, a parser will be
passed in.

The ``main()`` function is the entry point to be called from the CLI
or from Prefect.  The arguments should include the arguments defined
through the ArgumentParser, as well as any additional support for
Prefect (such as a logger argument).  For example::

  def main(obs_id=None, config_file=None, logger=None):
    ...

If you want the submodule to be executable directly as a script (or
through ``python -m``), add a ``__main__`` handling block like this
one::

  if __name__ == '__main__':
    util.main_launcher(main, get_parser)

To register a properly organized submodule in the ``so-site-pipeline``
command line wrapper, edit ``cli.py`` and see comments inline.

"""

import argparse

from . import (
    analyze_bright_ptsrc,
    check_book,
    make_source_flags,
    make_uncal_beam_map,
    update_g3tsmurf_db,
    preprocess_tod,
    update_obsdb,
    make_ml_map,
)

# Dictionary matching element name to a submodule (which must have
# been imported above).  Note for these to work, the submodule must
# have defined a get_parser() and a main() function as described in
# the module comments above.

ELEMENTS = {
    'analyze-bright-ptsrc': analyze_bright_ptsrc,
    'check-book': check_book,
    'make-source-flags': make_source_flags,
    'make-uncal-beam-map': make_uncal_beam_map,
    'update-g3tsmurf-db': update_g3tsmurf_db,
    'preprocess-tod': preprocess_tod,
    'update-obsdb': update_obsdb,
    'make-ml-map': make_ml_map,
}

CLI_NAME = 'so-site-pipeline'

def get_parser():
    parser = argparse.ArgumentParser()
    # Make sure all args here are redirected to vars starting with
    # '_'.  We are going to clean those off before passing to the
    # subcommand.
    parser.add_argument('--bash-completion', action='store_true',
                        dest='_bash_completion')
    sps = parser.add_subparsers(dest='_pipemod')

    for name, module in ELEMENTS.items():
        sp = sps.add_parser(name)
        module.get_parser(sp)

    return parser

def main():
    parser = get_parser()
    args = parser.parse_args()

    # Extract top-level args ...
    top_args = {k: v for k, v in vars(args).items()
                if k[0] == '_'}
    [delattr(args, k) for k in top_args]
    top_args = argparse.Namespace(**top_args)

    if top_args._bash_completion:
        names = list(ELEMENTS.keys())
        print('complete -W "%s" %s' % (' '.join(names), CLI_NAME))
        parser.exit()

    if top_args._pipemod is None:
        parser.error('First argument must be a sub-command.')

    module = ELEMENTS[top_args._pipemod]
    module.main(**vars(args))


import setuptools

with open('VERSION.txt', 'r') as f:
    version = f.read().strip()

setuptools.setup(
    name="odoo-addons-oca-stock-logistics-transport",
    description="Meta package for oca-stock-logistics-transport Odoo addons",
    version=version,
    install_requires=[
        'odoo-addon-shipment_advice>=16.0dev,<16.1dev',
        'odoo-addon-shipment_advice_planner>=16.0dev,<16.1dev',
        'odoo-addon-stock_dock>=16.0dev,<16.1dev',
    ],
    classifiers=[
        'Programming Language :: Python',
        'Framework :: Odoo',
        'Framework :: Odoo :: 16.0',
    ]
)


"""
Given two strings s and t of lengths m and n respectively, return
the minimum window substring of s such that every character in t
(including duplicates) is included in the window. If there is no
such substring, return the empty string "".

The testcases will be generated such that the answer is unique.

A substring is a contiguous sequence of characters within the string.

Example 1:

Input: s = "ADOBECODEBANC", t = "ABC"
Output: "BANC"
Explanation: The minimum window substring "BANC" includes 'A', 'B', and
'C' from string t.
"""

import collections

class Solution(object):
    def minWindow(self, s, t):
        """
        :type s: str
        :type t: str
        :rtype: str
        """
        left = 0
        right = 0

        min_window = ""
        min_len = float('inf')
        count = 0

        counter = collections.Counter(t)
        counter_search = collections.defaultdict(int)

        while right < len(s):
            counter_search[s[right]] += 1
            if s[right] in counter:
                if counter_search[s[right]] <= counter[s[right]]:
                    count += 1

            while left <= right and count == len(t):
                # check if this would be min window
                if min_len > right - left + 1:
                    min_len = right - left + 1
                    min_window = s[left: right+1]

                # move left pointer, if no longer satisfies, remove 1 from count
                counter_search[s[left]] -= 1
                if s[left] in counter and counter_search[s[left]] < counter[s[left]]:
                    count -= 1

                left += 1

            right += 1

        return min_window


#coding:utf-8
import tensorflow as tf
import logging
from data.data_reader_distill import DatasetReaderDistill
from tensorflow import set_random_seed
set_random_seed(12345)
if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    logger = logging.getLogger("brc")
    logger.setLevel(logging.INFO)
    brc_data = DatasetReaderDistill(train_file='./dataset/seg_train_data_20w.txt',
                             dev_file='./dataset/seg_dev_data_20w.txt',
                             # test_file ='./dataset/test_data'
                             )
    from data.vocab import Vocab

    vocab = Vocab(lower=True)
    import sys
    for word in brc_data.word_iter(None):
        vocab.add(word)
        for char in word:
            vocab.add_char(char)
    logger.info(' char size {}'.format(vocab.get_char_vocab_size()))
    logger.info(' vocab size {} '.format(vocab.get_word_vocab()))

    unfiltered_vocab_size = vocab.size()
    unfiltered_char_size = vocab.get_char_vocab_size()
    vocab.filter_tokens_by_cnt(min_cnt=2)
    vocab.filter_chars_by_cnt(min_cnt=2)

    import os
    if not os.path.exists('vocab.txt'):
        writer = open('vocab.txt','a+',encoding='utf-8')
        for word,id in vocab.token2id.items():
            writer.write(word+'\t'+str(id)+'\n')
        writer.close()

    filtered_num = unfiltered_vocab_size - vocab.size()
    logger.info('After filter {} tokens, the final vocab size is {}'.format(filtered_num,
                                                                            vocab.size()))

    filtered_num = unfiltered_char_size -vocab.get_char_vocab_size()
    logger.info('After filter {} tokens, the final vocab size is {}'.format(filtered_num,
                                                                            vocab.get_char_vocab_size()))
    # sys.exit(1)

    #vocab.load_pretrained_embeddings('/Users/apple/Downloads/embeddings/sgns.wiki.word')

    logger.info('after load embedding vocab size is {}'.format(vocab.size()))
    # print(vocab.embeddings.shape)
    # import sys
    # sys.exit(1)

    brc_data.convert_to_ids(vocab)
    import numpy as np
    for task_balance in np.arange(0.12,0.51,0.01):

        from model.text_cnn import TextCNN
        from model.abilstm import  ABLSTM
        from model.bcnn import  BCNN
        # from model.char_cnn import CharCNN
        from model.char_cnn2 import  CharCNN
        from model.bilstm import  BLSTM
        from model.multi_text_cnn import  MultiTextCNN
        from model.char_word_cnn import  CharTextCNN
        #model = CharCNN(vocab,num_class=2)
        #model = BCNN(vocab,num_class=2)
        #model = CharTextCNN(vocab,num_class=2)
        #model = ABLSTM(vocab,num_class=2)
        #model = BLSTM(vocab,num_class=2)
        tf.reset_default_graph()
        save_dir = '/Users/apple/Downloads/news_qa/checkpoint'
        model = TextCNN(vocab,num_class=3,task_balance=0.12,soft_temperature=10)
        model.compile(tf.train.AdamOptimizer, 0.001)
        model.load('/Users/apple/Downloads/news_qa/pretrained_checkpoint/best_weights/')
        model.train_and_evaluate(brc_data, evaluator=None, epochs=5, save_dir=save_dir)
        sys.exit(1)

        if task_balance==1:
            model.train_and_evaluate(brc_data,evaluator=None,epochs=5,save_dir=save_dir)
        else:
            model.load('/Users/apple/Downloads/news_qa/checkpoint/best_weights/')
            model.train_and_evaluate(brc_data,evaluator=None,epochs=5,save_dir=save_dir)

        print('..........finish training with  {} ............'.format(task_balance))

    # from model.bilstm import  BLSTM
    # model = BLSTM(vocab)
    # model.compile(tf.train.AdamOptimizer, 0.001)
    # model.train_and_evaluate(brc_data,evaluator=None,epochs=15)


from django.contrib import admin
from .models import Project, Link, Tag, Screenshot
from django_markdown.admin import AdminMarkdownWidget
from django.db import models

class LinkInline(admin.StackedInline):
    model= Link
    extra=1
class ScreenInline(admin.StackedInline):
    model= Screenshot
    extra=1
class ProjectAdmin(admin.ModelAdmin):
    formfield_overrirdes= {models.TextField:  {'widget': AdminMarkdownWidget}}
    inlines=[LinkInline,ScreenInline]

# Register your models here.

admin.site.register(Project, ProjectAdmin)
admin.site.register(Tag)


import os

import pytest

from core.models import Dataset, Project, DataDeclaration
from test import factories
from django.conf import settings
from json import loads
from importlib import import_module

@pytest.mark.parametrize(
    "validator_class",
    [
        "DatasetJSONSchemaValidator",
        "ProjectJSONSchemaValidator",
        "InstitutionJSONSchemaValidator",
    ],
)
def test_json_schema_version(
    celery_session_worker, can_defer_constraint_checks, validator_class
):
    validator_class = getattr(
        import_module("core.importer.JSONSchemaValidator"), validator_class
    )
    validator = validator_class()
    schema_filename = os.path.join(validator.base_path, validator.schema_name)
    with open(schema_filename) as f:
        schema = loads(f.read())
    assert os.path.dirname(schema.get("$id")) == os.path.dirname(
        settings.IMPORT_JSON_SCHEMAS_URI
    )

@pytest.mark.skip("TBD")
def test_json_schema_validation():
    # Test validators on demo data
    pass


#!/usr/bin/env python3

import struct
from optparse import OptionParser

import serial

signal2AmplAddr = 0
signalAmplAddr = 2
signalImagAddr = 4
signalRealAddr = 6

signal2Ampl = 0
signalAmpl = 0
signalImag = 0
signalReal = 0

# configure the serial connections (the parameters differs on the device you are connecting to)
ser = serial.Serial(port='/dev/ttyUSB0', baudrate=115200, bytesize=8, parity='N', stopbits=1, timeout=None)

def readWord(addr):
    command = 0b00010000
    binData = struct.pack('>BBHB', 0, command, addr, 2)
    ser.write(binData)
    data = ser.read(2)
    return int.from_bytes(data, byteorder='big', signed=True)

# create function for readAll
signalAmpl = readWord(signalAmplAddr)
signal2Ampl = readWord(signal2AmplAddr)
signalReal = readWord(signalRealAddr)
signalImag = readWord(signalImagAddr)

print("F1Ampl = " + str(signalAmpl))
print("F2Ampl = " + str(signal2Ampl))
print()
print("F1Real = " + str(signalReal))
print("F1Imag = " + str(signalImag))
print()

ser.close()


import torch
import argparse
from Nets import SineRnn
import torch.nn as nn
import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

parser = argparse.ArgumentParser()
parser.add_argument('--lr', dest='learn_rate', default=0.1, type=int)
parser.add_argument('--epochs', dest='epochs', default=20, type=int)
args = parser.parse_args()
learn_rate = args.learn_rate
epochs = args.epochs

if torch.cuda.is_available():
    device = torch.device('cuda')
else:
    device = torch.device('cpu')

# 画线
def draw(y, test, i):
    plt.figure(figsize = (40,20))
    plt.xlabel('x')
    plt.ylabel('y')
    plt.xticks()
    plt.yticks()
    plt.plot(np.arange(999), test[0][:999], linewidth=2.0, color = 'blue')
    plt.plot(np.arange(999), test[1][:999], linewidth=2.0, color = 'yellow')
    plt.plot(np.arange(999), test[2][:999], linewidth=2.0, color = 'red')
    plt.plot(np.arange(999, 1999), y[0][999:], linestyle=':', linewidth=2.0, color = 'blue')
    plt.plot(np.arange(999, 1999), y[1][999:], linestyle=':', linewidth=2.0, color = 'yellow')
    plt.plot(np.arange(999, 1999), y[2][999:], linestyle=':', linewidth=2.0, color = 'red')
    # print(i)
    plt.savefig('predict' + str(i) + '.jpg' )
    plt.close()

# 反向传播，利用 LBFGS 优化器进行优化
def closure():
    optimizer.zero_grad()
    out = model(input)
    loss = lossfunc(out, target).to(device)
    print('\tloss:', loss.item())
    loss.backward()
    return loss

def train():
    for epoch in range(epochs):
        print('Epoch  ' + str(epoch + 1) + ': ')
        optimizer.step(closure)
        test(epoch + 1)

def test(id):
    with torch.no_grad():
        predict = 1000
        pred = model(test_input, predict=predict)
        loss = lossfunc(pred[:, :-predict], test_target)
        print('\ttest loss:', loss.item())
        y = pred.detach().cpu().numpy()
    draw(y, test_target.cpu().numpy(), id)

data = torch.load('./traindata.pt')
input = torch.from_numpy(data[3:, :-1]).to(device)
target = torch.from_numpy(data[3:, 1:]).to(device)
test_input = torch.from_numpy(data[:3, :-1]).to(device)
test_target = torch.from_numpy(data[:3, 1:]).to(device)

lossfunc = nn.MSELoss()
model = SineRnn(in_features=1, hide_features=20, out_features=1).to(device).double()
optimizer = torch.optim.LBFGS(model.parameters(), lr=learn_rate)
train_data = torch.from_numpy(torch.load('traindata.pt')).to(device)
train()
model.eval()


import cv2
import os

modelFile = "res10_300x300_ssd_iter_140000.caffemodel"
configFile = "deploy.prototxt.txt"  # lines 4-12 configure the DNN face detection model to be used
net = cv2.dnn.readNetFromCaffe(configFile, modelFile)

class Queue:
    def __init__(self):
        self.items = []

    def isEmpty(self):
        return self.items == []

    def enqueue(self, item):
        self.items.insert(0, item)

    def dequeue(self):
        if not self.isEmpty():
            return self.items.pop()

class Image:
    def __init__(self, img, name, savelocation):
        self.image = img
        self.name = name
        self.saveLoc = savelocation

    def extractFace(self, con=0.8):
        print("EXTRACTING FACE...")
        frameHeight = self.image.shape[0]  # load in image and get images height and width
        frameWidth = self.image.shape[1]
        blob = cv2.dnn.blobFromImage(self.image, 1.0, (300, 300), [104, 117, 123], False,
                                     False)  # create a blob from the image
        net.setInput(blob)
        detections = net.forward()  # give blob as input to neural network and pass it through net and store output from net in detections
        for i in range(detections.shape[2]):  # iterates over number of faces detected
            confidence = detections[0, 0, i, 2]  # gets the confidence from model that this is a face
            if confidence >= con:  # if confidence higher than 0.8
                print('FACE DETECTED IN ' + str(self.name))
                x1 = int(detections[0, 0, i, 3] * frameWidth)
                y1 = int(detections[0, 0, i, 4] * frameHeight)  # gets coordinates of box that bounds the face
                x2 = int(detections[
                             0, 0, i, 5] * frameWidth)  # coordinates are normalised between 0 and 1 so need to be multiplied by image height and width
                y2 = int(detections[0, 0, i, 6] * frameHeight)
                face = self.image[y1:y2, x1:x2]  # creates a sub image using original image and co-ordinates of face
                face = cv2.resize(face, (224, 224))
                cv2.imwrite(os.path.join(self.saveLoc, self.name), face)  # saves face to face folder

class Video:
    def __init__(self, address, node, level):
        self.videoAddress = address
        self.video = cv2.VideoCapture(self.videoAddress)
        self.node = node
        self.level = level

    def nameFrame(self, count, fps):
        if len(str(count // int(fps))) == 1:
            num = '000' + str(count // 30)
        elif len(str(count // int(fps))) == 2:
            num = '00' + str(count // int(fps))
        elif len(str(count // int(fps))) == 3:
            num = '0' + str(count // 30)
        else:
            num = str(count // 30)
        name = (str(self.node) + '-' + str(self.level) + '-' + num + '.jpg')
        return name

    def extractFrames(self, saveLoc, fps=30):
        frameQueue = Queue()
        count = 1
        success = 1
        while success:
            success, frame = self.video.read()
            if count % int(fps) == 0:
                frameName = self.nameFrame(count, fps)
                frameQueue.enqueue(frame)
            count += 1
        self.video.release()
        while not frameQueue.isEmpty():
            image = Image(frameQueue.dequeue(), frameName, saveLoc)
            image.extractFace()

# main subroutine takes in a 2D array of directories for each fold and the participants within that directory
def main(directories):
    levels = [0, 10]  # array holding levels we want to extract images for
    current = 0  # represents index of current directory we are working through from the parameter array
    for directory in directories:  # iterates through each directory
        for participants in directories[current][1]:  # iterates through each participant in each directory
            for level in levels:  # iterates through each level
                if level == 10:
                    if int(participants) <= 39:
                        dir = str(directory[0]) + '/' + str(participants) + '/'
                        for filename in os.listdir(dir):
                            lev = filename.split('.')
                            if lev[0] == '10':
                                address = dir + filename
                                print(address)
                                vid = Video(address, participants, level)
                                vid.extractFrames('C:/Users/Hadi/PycharmProjects/NEA/Faces/Train/10')
                    elif int(participants) > 39 and int(participants) <= 48:
                        dir = str(directory[0]) + '/' + str(participants) + '/'
                        for filename in os.listdir(dir):
                            lev = filename.split('.')
                            if lev[0] == '10':
                                address = dir + filename
                                print(address)
                                vid = Video(address, participants, level)
                                vid.extractFrames('C:/Users/Hadi/PycharmProjects/NEA/Faces/Valid/10')
                    else:
                        dir = str(directory[0]) + '/' + str(participants) + '/'
                        for filename in os.listdir(dir):
                            lev = filename.split('.')
                            if lev[0] == '10':
                                address = dir + filename
                                print(address)
                                vid = Video(address, participants, level)
                                vid.extractFrames('C:/Users/Hadi/PycharmProjects/NEA/Faces/Test/10')
                elif level == 0:
                    if int(participants) <= 39:
                        dir = str(directory[0]) + '/' + str(participants) + '/'
                        for filename in os.listdir(dir):
                            lev = filename.split('.')
                            if lev[0] == '0':
                                address = dir + filename
                                print(address)
                                vid = Video(address, participants, level)
                                vid.extractFrames('C:/Users/Hadi/PycharmProjects/NEA/Faces/Train/0')
                    elif int(participants) > 39 and int(participants) <= 48:
                        dir = str(directory[0]) + '/' + str(participants) + '/'
                        for filename in os.listdir(dir):
                            lev = filename.split('.')
                            if lev[0] == '0':
                                address = dir + filename
                                vid = Video(address, participants, level)
                                vid.extractFrames('C:/Users/Hadi/PycharmProjects/NEA/Faces/Valid/0')
                    else:
                        dir = str(directory[0]) + '/' + str(participants) + '/'
                        for filename in os.listdir(dir):
                            lev = filename.split('.')
                            if lev[0] == '0':
                                address = dir + filename
                                print(address)
                                vid = Video(address, participants, level)
                                vid.extractFrames('C:/Users/Hadi/PycharmProjects/NEA/Faces/Test/0')
        current += 1

# main([['F:/Fold1_part1',['01','02','03','04','05','06']],['F:/Fold1_part2',['07','08','09','10','11','12']],['G:/Fold2_part1',['13','14','15','16','17','18']],['G:/Fold2_part2',['19','20','21','22','23','24']],['C:/Users/Hadi/PycharmProjects/NEA/RawVideos/Fold3_part1',['25','26','27','28','29','30']],['C:/Users/Hadi/PycharmProjects/NEA/RawVideos/Fold3_part2',['31','32','33','34','35','36']],['G:/Fold4_part1',['37','38','39','40','42']],['G:/Fold4_part2',['43','44','45','46','47','48']],['F:/Fold5_part1',['49','50','51','52','53','54']],['C:/Users/Hadi/PycharmProjects/NEA/RawVideos/Fold5_part2',['55','56','57','58','59','60']]])
main([['C:/Users/Hadi/PycharmProjects/NEA/RawVideos/Fold5_part2', ['55', '56', '57', '58', '59', '60']]])


import os
from pathlib import Path
from typing import Union

import yaml

from ofm.defaults import PROJECT_DIR

class Settings:
    def __init__(
        self,
        root_dir: Union[str, Path] = PROJECT_DIR,
        settings: Union[str, Path] = os.path.join(PROJECT_DIR, "settings.yaml"),
    ) -> None:
        self.root_dir = root_dir
        self.res: str = os.path.join(root_dir, "res")
        self.images: str = os.path.join(root_dir, "images")
        self.db: str = os.path.join(self.res, "db")
        self.save: str = os.path.join(root_dir, "save")
        self.clubs_def: str = os.path.join(self.res, "clubs_def.json")
        self.fifa_codes: str = os.path.join(self.res, "fifa_country_codes.json")
        self.fifa_conf: str = os.path.join(self.res, "fifa_confederations.json")
        self.squads_def: str = os.path.join(self.res, "squads_def.json")
        self.squads_file: str = os.path.join(self.db, "squads.json")
        self.players_file: str = os.path.join(self.db, "players.json")
        self.clubs_file: str = os.path.join(self.db, "clubs.json")
        self.settings_file: str = settings

    def get_data(self) -> dict:
        return {
            "res": self.res,
            "images": self.images,
            "db": self.db,
            "save": self.save,
            "clubs_def": self.clubs_def,
            "fifa_codes": self.fifa_codes,
            "fifa_conf": self.fifa_conf,
            "squads": self.squads_file,
            "players": self.players_file,
            "clubs": self.clubs_file,
        }

    def parse_settings(self, data: dict) -> None:
        self.res = data["res"]
        self.images = data["images"]
        self.db = data["db"]
        self.save = data["save"]
        self.clubs_def = data["clubs_def"]
        self.fifa_codes = data["fifa_codes"]
        self.fifa_conf = data["fifa_conf"]
        self.squads_file = data["squads"]
        self.players_file = data["players"]
        self.clubs_file = data["clubs"]

    def load_settings(self) -> None:
        with open(self.settings_file, "r") as fp:
            data = yaml.safe_load(fp)
            self.parse_settings(data)

    def create_settings(self) -> None:
        with open(self.settings_file, "w") as fp:
            yaml.safe_dump(self.get_data(), fp)

    def get_settings(self) -> None:
        if os.path.exists(self.settings_file):
            self.load_settings()
        else:
            self.create_settings()


import random
import cv2
import numpy as np

class Image:
    def __init__(self,img):
        self.img = img
        self.points = []
        self.defaultOrder()

    def incr(self,r, c, amount):
        if r >= self.img.shape[0] or c >= self.img.shape[1] \
            or r < 0 or c < 0:
            return

        self.img[r,c][0] = (self.img[r,c][0] + amount) % 256

    def incg(self,r, c, amount):
        if r >= self.img.shape[0] or c >= self.img.shape[1] \
            or r < 0 or c < 0:
            return

        self.img[r,c][1] = (self.img[r,c][1] + amount) % 256

    def incb(self,r, c, amount):
        if r >= self.img.shape[0] or c >= self.img.shape[1] \
            or r < 0 or c < 0:
            return

        self.img[r,c][2] = (self.img[r,c][2] + amount) % 256

    # returns default ordering of points (as if they were
    # text on a page)
    def defaultOrder(self):
        points = []
        for r in range(self.img.shape[0]):
            for c in range(self.img.shape[1]):
                points.append([r,c])
        self.points = points

    # returns outwards circle progression ordering of points
    # dradius is the width of the bands
    def circleOrder(self,dradius):
        points = []

        r_bar = self.img.shape[0] // 2
        c_bar = self.img.shape[1] // 2
        radius = dradius
        while radius < .5*(self.img.shape[0]**2+self.img.shape[1]**2)**.5:
            for r in range(self.img.shape[0]):
                for c in range(self.img.shape[1]):
                    point_rad = ((r - r_bar)**2 + (c - c_bar)**2)**.5
                    if point_rad < radius and point_rad > radius - dradius:
                        points.append([r,c])
            radius += dradius

        self.points = points

    # set all points in image to black
    def black(self):
        for r in range(self.img.shape[0]):
            for c in range(self.img.shape[1]):
                self.img[r,c][0] = 0
                self.img[r,c][1] = 0
                self.img[r,c][2] = 0

    # applies random walk filter over points
    # filter has mean mu and variance sigma^2
    # filter is applied over image; apply black() before for pure RW
    def randomWalk(self, mu, sigma):

        n = len(self.points)

        # generate init residuals
        rfilter = int(np.random.normal(mu,sigma)) % 256
        gfilter = int(np.random.normal(mu,sigma)) % 256
        bfilter = int(np.random.normal(mu,sigma)) % 256

        # generate rest of filters
        for i in range(n):
            self.incr(self.points[i][0], self.points[i][1], rfilter)
            self.incg(self.points[i][0], self.points[i][1], gfilter)
            self.incb(self.points[i][0], self.points[i][1], bfilter)
            rfilter = (rfilter + int(np.random.normal(0,sigma))) % 256
            gfilter = (gfilter + int(np.random.normal(0,sigma))) % 256
            bfilter = (bfilter + int(np.random.normal(0,sigma))) % 256

    # saturates image. I forget which values of s work well...
    def sat(self, s):
        for r in range(self.img.shape[0]):
            for c in range(self.img.shape[1]):
                if self.img[r,c][0] < 128:
                    self.img[r,c][0] -= int((128 - self.img[r,c][0]) ** s)
                else:
                    self.img[r,c][0] += int((self.img[r,c][0] - 128) ** s)
                if self.img[r,c][1] < 128:
                    self.img[r,c][1] -= int((128 - self.img[r,c][1]) ** s)
                else:
                    self.img[r,c][1] += int((self.img[r,c][1] - 128) ** s)
                if self.img[r,c][2] < 128:
                    self.img[r,c][2] -= int((128 - self.img[r,c][2]) ** s)
                else:
                    self.img[r,c][2] += int((self.img[r,c][2] - 128) ** s)

image = cv2.imread('wp2try.jpg')
img = Image(image)

img.circleOrder(3)

#img.black()
img.randomWalk(150,.5)

#img.sat(.5)
cv2.imwrite('new.png', img.img)


import os
from queue import Queue, Empty
from shutil import rmtree
import sys
from tempfile import gettempdir
from typing import Optional
import threading
import time
import unittest
from urllib.parse import quote

# Related third party imports
from genicam.gentl import TimeoutException
import numpy as np

# Local application/library specific imports
from harvesters.test.base_harvester import TestHarvesterCoreBase
from harvesters.test.base_harvester import get_cti_file_path
from harvesters.core import Callback
from harvesters.core import Harvester
from harvesters.core import ImageAcquirer
from harvesters.test.helper import get_package_dir
from harvesters.util.pfnc import Dictionary
from harvesters.core import Component2DImage
from harvesters.util.pfnc import Mono8, Mono10, Mono12, Mono14, Mono16
from harvesters.util.pfnc import Mono10Packed, Mono12Packed
from harvesters.util.pfnc import Mono10p, Mono12p, Mono14p

class TestHarvesterCore(TestHarvesterCoreBase):
    sleep_duration = .5  # Time to keep sleeping [s]

    def test_basic_usage_1(self):
        """
        We walk through a basic usage and manually close the image
        acquirer.

        :return: None.
        """

        # Prepare an image acquirer for device #0.
        ia = self.harvester.create_image_acquirer(0)

        # Acquire images.
        self._basic_usage(ia)

        # Discard the image acquirer.
        ia.destroy()

    def test_basic_usage_2(self):
        """
        We walk through a basic usage and omit manually closing the image
        acquirer using the with statement.

        :return: None.
        """

        # Prepare an image acquirer for device #0.
        with self.harvester.create_image_acquirer(0) as ia:

            # Acquire images.
            self._basic_usage(ia)

    def _basic_usage(self, ia: ImageAcquirer):
        # Start image acquisition.
        ia.start_acquisition()

        # Fetch a buffer that is filled with image data.
        with ia.fetch_buffer() as buffer:
            # Reshape it.
            self._logger.info('{0}'.format(buffer))

        # Stop image acquisition.
        ia.stop_acquisition()

    def test_multiple_image_acquirers(self):
        num_devices = len(self.harvester.device_info_list)
        self._test_image_acquirers(num_ias=num_devices)

    def _test_image_acquirers(self, num_ias=1):
        #
        self._logger.info('Number of devices: {0}'.format(num_ias))

        #
        ias = []  # Image Acquirers

        #
        for list_index in range(num_ias):
            ias.append(
                self.harvester.create_image_acquirer(
                    list_index=list_index
                )
                # Or you could simply do the same thing as follows:
                # self.harvester.create_image_acquirer(list_index)
            )

        #
        for i in range(3):
            #
            self._logger.info('---> Round {0}: Set up'.format(i))
            for index, ia in enumerate(ias):
                ia.start_acquisition()
                self._logger.info(
                    'Device #{0} has started image acquisition.'.format(index)
                )

            k = 0

            # Run it as fast as possible.
            frames = 10

            while k < frames:
                for ia in ias:
                    if k % 2 == 0:
                        # Option 1: This way is secure and preferred.
                        try:
                            # We know we've started image acquisition but this
                            # try-except block is demonstrating a case where
                            # a client called fetch_buffer method even though
                            # he'd forgotten to start image acquisition.
                            with ia.fetch_buffer() as buffer:
                                self._logger.info('{0}'.format(buffer))
                        except AttributeError:
                            # Harvester Core has not started image acquisition
                            # so calling fetch_buffer() raises AttributeError
                            # because None object is used for the with
                            # statement.
                            pass
                    else:
                        # Option 2: You can manually do the same job but not
                        # recommended because you might forget to queue the
                        # buffer.
                        buffer = ia.fetch_buffer()
                        self._logger.info('{0}'.format(buffer))

                #
                k += 1

            #
            self._logger.info('<--- Round {0}: Tear down'.format(i))
            for index, ia in enumerate(ias):
                ia.stop_acquisition()
                self._logger.info(
                    'Device #{0} has stopped image acquisition.'.format(index)
                )

        for ia in ias:
            ia.destroy()

    def test_controlling_a_specific_camera(self):
        if not self.is_running_with_default_target():
            return

        # The basic usage.
        ia = self.harvester.create_image_acquirer(0)
        ia.destroy()

        # The basic usage but it explicitly uses the parameter name.
        ia = self.harvester.create_image_acquirer(
            list_index=0
        )
        ia.destroy()

        # The key can't specify a unique device so it raises an exception.
        with self.assertRaises(ValueError):
            self.harvester.create_image_acquirer(
                vendor='EMVA_D'
            )

        # The key specifies a unique device.
        self._logger.info(self.harvester.device_info_list)
        ia = self.harvester.create_image_acquirer(
            serial_number='SN_InterfaceA_0'
        )
        ia.destroy()

    def test_timeout_on_fetching_buffer(self):
        if not self.is_running_with_default_target():
            return

        # Create an image acquirer:
        ia = self.harvester.create_image_acquirer(0)

        # We do not start image acquisition:
        #ia.start_acquisition()

        timeout = 3  # sec

        with self.assertRaises(TimeoutException):
            # Try to fetch a buffer but the IA will immediately raise
            # TimeoutException because it's not started image acquisition:
            _ = ia.fetch_buffer(timeout=timeout)

        # Then we setup the device for software trigger mode:
        ia.remote_device.node_map.TriggerMode.value = 'On'
        ia.remote_device.node_map.TriggerSource.value = 'Software'

        # We're ready to start image acquisition:
        ia.start_acquisition()

        with self.assertRaises(TimeoutException):
            # Try to fetch a buffer but the IA will raise TimeoutException
            # because we've not triggered the device so far:
            _ = ia.fetch_buffer(timeout=timeout)

        # We finally acquire an image triggering the device:
        buffer = None
        self.assertIsNone(buffer)

        ia.remote_device.node_map.TriggerSoftware.execute()
        buffer = ia.fetch_buffer(timeout=timeout)
        self.assertIsNotNone(buffer)
        self._logger.info('{0}'.format(buffer))
        buffer.queue()

        # Now we stop image acquisition:
        ia.stop_acquisition()
        ia.destroy()

    def test_stop_start_and_stop(self):
        # Create an image acquirer:
        ia = self.harvester.create_image_acquirer(0)

        # It's not necessary but we stop image acquisition first;
        #
        ia.stop_acquisition()

        # Then start it:
        ia.start_acquisition()

        # Fetch a buffer to make sure it's working:
        with ia.fetch_buffer() as buffer:
            self._logger.info('{0}'.format(buffer))

        # Then stop image acquisition:
        ia.stop_acquisition()

        # And destroy the ImageAcquirer:
        ia.destroy()

    def test_num_holding_filled_buffers(self):
        if not self.is_running_with_default_target():
            return

        # Connect to the first camera in the list:
        self.ia = self.harvester.create_image_acquirer(0)

        # Setup the camera before starting image acquisition:
        self.setup_camera()

        #
        tests = [
            self._test_issue_120_1
        ]
        for test in tests:
            for num_images in [1, 2, 4, 8]:
                #
                self.ia.num_filled_buffers_to_hold = num_images

                # Start image acquisition:
                self.ia.start_acquisition(run_in_background=True)

                # Run a test:
                test(num_images)

                # Stop image acquisition:
                self.ia.stop_acquisition()

    def _test_issue_120_1(self, num_images):
        # Make sure num_holding_filled_buffers is incremented every trigger:
        for i in range(num_images):
            #
            self.assertEqual(
                self.ia.num_holding_filled_buffers, i
            )

            # Trigger it:
            self.generate_software_trigger(sleep_s=self.sleep_duration)

            # It must be incremented:
            self.assertEqual(
                self.ia.num_holding_filled_buffers, i + 1
            )

        # Trigger it again, we know it's redundant compared to the
        # maximum capacity:
        self.generate_software_trigger(sleep_s=self.sleep_duration)

        # Make sure num_holding_filled_buffers does not exceed
        # num_filled_buffers_to_hold:
        self.assertEqual(
            self.ia.num_filled_buffers_to_hold,
            self.ia.num_holding_filled_buffers
        )

        # Make sure num_holding_filled_buffers is decreased every time
        # a filled buffer is fetched:
        for i in range(num_images):
            #
            self.assertEqual(
                self.ia.num_holding_filled_buffers,
                num_images - i
            )
            #
            with self.ia.fetch_buffer():
                #
                self.assertEqual(
                    self.ia.num_holding_filled_buffers,
                    num_images - (i + 1)
                )

    def setup_camera(self):
        self.ia.remote_device.node_map.AcquisitionMode.value = 'Continuous'
        self.ia.remote_device.node_map.TriggerMode.value = 'On'
        self.ia.remote_device.node_map.TriggerSource.value = 'Software'

    def generate_software_trigger(self, sleep_s=0.):
        # Trigger the camera because you have already setup your
        # equipment for the upcoming image acquisition.
        self.ia.remote_device.node_map.TriggerSoftware.execute()
        #
        if sleep_s > 0:
            time.sleep(sleep_s)

    def test_issue_59(self):
        if not self.is_running_with_default_target():
            return

        # Connect to the first camera in the list.
        self.ia = self.harvester.create_image_acquirer(0)

        #
        min = self.ia._data_streams[0].buffer_announce_min
        with self.assertRaises(ValueError):
            self.ia.num_buffers = min - 1

        #
        with self.assertRaises(ValueError):
            self.ia.num_filled_buffers_to_hold = 0

        #
        self.ia.num_filled_buffers_to_hold = min

    def test_issue_60(self):
        if not self.is_running_with_default_target():
            return

        # Connect to the first camera in the list.
        self.ia = self.harvester.create_image_acquirer(0)

        # Check the number of buffers:
        self.assertEqual(16, self.ia.num_buffers)

    @unittest.skip('It has been obsolete; see issue #141.')
    def test_issue_61(self):
        if not self.is_running_with_default_target():
            return

        # Connect to the first camera in the list.
        self.ia = self.harvester.create_image_acquirer(0)

        # Register a call back method:
        self.ia.on_new_buffer_arrival = self._callback_on_new_buffer_arrival

        # We turn software trigger on:
        self.setup_camera()

        # We have not fetched any buffer:
        self.assertEqual(0, len(self._buffers))

        # Start image acquisition:
        self.ia.start_acquisition(run_in_background=True)

        # Trigger the target device:
        num_images = self.ia.num_buffers
        self.assertTrue(num_images > 0)

        # Trigger the target device:
        for _ in range(num_images):
            self.generate_software_trigger(sleep_s=self.sleep_duration)

        # If the callback method was called, then we should have the same
        # number of buffers with num_images:
        self.assertEqual(num_images, len(self._buffers))

        # Release the buffers before stopping image acquisition:
        for buffer in self._buffers:
            buffer.queue()

        self._buffers.clear()

        # Then stop image acquisition:
        self.ia.stop_acquisition()

    def _callback_on_new_buffer_arrival(self):
        # Fetch a buffer and keep it:
        self._buffers.append(self.ia.fetch_buffer())

    def test_issue_66(self):
        if not self.is_running_with_default_target():
            return

        file_names = ['altered_plain.xml', 'altered_zip.zip']
        expected_values = ['plain', 'zip']
        for i, file_name in enumerate(file_names):
            self._test_issue_66(
                'issue_66_' + file_name, expected_values[i]
            )

    def _test_issue_66(self, file_name, expected_value):
        #
        xml_dir = self._get_xml_dir()

        # Connect to the first camera in the list.
        self.ia = self.harvester.create_image_acquirer(
            0, file_path=os.path.join(xml_dir, file_name)
        )

        # Compare DeviceModelNames:
        self.assertEqual(
            'Altered TLSimu (' + expected_value + ')',
            self.ia.remote_device.node_map.DeviceModelName.value
        )

        #
        self.ia.destroy()

    def test_issue_67(self):
        if not self.is_running_with_default_target():
            return

        file_names = ['altered_plain.xml', 'altered_zip.zip']
        for i, file_name in enumerate(file_names):
            self._test_issue_67(
                'issue_67_' + file_name
            )

    def _test_issue_67(self, expected_file_name):
        #
        xml_dir = self._get_xml_dir()

        #
        url = 'file://'
        file_path = xml_dir + '/' + expected_file_name

        # '\' -> '/'
        file_path.replace('\\', '/')

        # ':' -> '|'
        file_path.replace(':', '|')

        # ' ' -> '%20'
        file_path = quote(file_path)

        #
        url += file_path

        # Parse the URL:
        retrieved_file_path = ImageAcquirer._retrieve_file_path(url=url)

        # Compare file names:
        self.assertEqual(
            os.path.basename(retrieved_file_path),
            expected_file_name
        )

    def test_issue_121(self):
        #
        expected_file_path = '/Foo.xml'

        #
        url = 'file://' + expected_file_path
        retrieved_file_path = ImageAcquirer._retrieve_file_path(url=url)

        # Compare file names:
        self.assertEqual(
            retrieved_file_path,
            expected_file_path
        )

    @staticmethod
    def _get_xml_dir():
        return os.path.join(get_package_dir('harvesters'), 'test', 'xml')

    def test_issue_70(self):
        if not self.is_running_with_default_target():
            return

        # Connect to the first camera in the list:
        self.ia = self.harvester.create_image_acquirer(0)

        # Then check the minimum buffer number that a client can ask
        # the ImageAcquire object to prepare:
        self.assertEqual(5, self.ia.min_num_buffers)

    def test_issue_78(self):
        if not self.is_running_with_default_target():
            return

        # The device_info_list must not turn empty even if a given key
        # does not match to any candidate:
        self._logger.info(self.harvester.device_info_list)
        device_info_list = self.harvester.device_info_list.copy()
        try:
            self.harvester.create_image_acquirer(
                serial_number='abcdefghijklmnopqrstuwxyz!#$%&=~|<>'
            )
        except ValueError:
            self.assertEqual(
                device_info_list, self.harvester.device_info_list
            )

    def test_issue_130_1(self):
        #
        self.ia = self.harvester.create_image_acquirer(0)
        #
        self.ia.start_acquisition(run_in_background=False)
        #
        with self.ia.fetch_buffer() as buffer:
            self.assertIsNotNone(buffer)
        #
        self.ia.stop_acquisition()

    def test_issue_141(self):
        if not self.is_running_with_default_target():
            return

        # Connect to the first camera in the list.
        self.ia = self.harvester.create_image_acquirer(0)

        # We turn software trigger on:
        self.setup_camera()

        # Create a callback:
        self.on_new_buffer_available = _OnNewBufferAvailable(
            ia=self.ia, buffers=self._buffers
        )
        self.on_return_buffer_now = _OnReturnBufferNow(
            holder=self.on_new_buffer_available
        )

        # Trigger the target device:
        self.num_images = self.ia.num_buffers
        self.assertTrue(self.num_images > 0)

        #
        tests = [
            self._test_141_with_callback, self._test_141_without_callback
        ]
        for test in tests:
            # We have not yet fetched any buffer:
            self.assertEqual(0, len(self.on_new_buffer_available.buffers))

            # Run a sub-test:
            test()

            # Then stop image acquisition:
            self.ia.stop_acquisition()

    def _test_141_with_callback(self):
        # Add it to the image acquire so that it can get notified when the
        # event happened:
        self.ia.add_callback(
            ImageAcquirer.Events.NEW_BUFFER_AVAILABLE,
            self.on_new_buffer_available
        )
        self.ia.add_callback(
            ImageAcquirer.Events.RETURN_ALL_BORROWED_BUFFERS,
            self.on_return_buffer_now
        )

        #
        self._test_141_body()

        # If the callback method was called, then we should have the same
        # number of buffers with num_images:
        self.assertEqual(
            self.ia.num_buffers, len(self.on_new_buffer_available.buffers)
        )

    def _test_141_without_callback(self):
        # Remove all callbacks to not any callback work:
        self.ia.remove_callbacks()

        #
        self._test_141_body()

        # The list must be empty because the emit method has not been called:
        self.assertEqual(
            0, len(self.on_new_buffer_available.buffers)
        )

    def _test_141_body(self):
        # Start image acquisition:
        self.ia.start_acquisition(run_in_background=True)

        # Trigger the target device:
        for _ in range(self.num_images):
            self.generate_software_trigger(sleep_s=self.sleep_duration)

    def test_issue_150(self):
        # Create an image acquirer:
        self.ia = self.harvester.create_image_acquirer(0)

        # TLSimu does not support SingleFrame in reality:
        # modes = ["SingleFrame", "Continuous"]

        # Test only "Continuous":
        modes = ["Continuous"]
        for mode in modes:
            self.ia.remote_device.node_map.AcquisitionMode.value = mode

            for i in range(32):
                # Then start it:
                self.ia.start_acquisition()

                # Fetch a buffer to make sure it's working:
                with self.ia.fetch_buffer() as buffer:
                    self._logger.info('{0}'.format(buffer))

            # Then stop image acquisition:
            self.ia.stop_acquisition()

        # And destroy the ImageAcquirer:
        self.ia.destroy()

    def test_issue_146(self):
        #
        tests = [
            self._test_issue_146_group_packed_10,
            self._test_issue_146_group_packed_12,
            self._test_issue_146_packed_10,
            self._test_issue_146_packed_12,
            self._test_issue_146_mono_unpacked_multibytes,
        ]
        #
        for test in tests:
            test()

    def _test_issue_146_group_packed_10(self):
        _1st = 0xff
        _3rd = 0xff
        ba = bytes([_1st, 0x33, _3rd])
        packed = np.frombuffer(ba, dtype=np.uint8)
        pf = Dictionary.get_proxy('BayerRG10Packed')
        unpacked = pf.expand(packed)
        self.assertEqual(_1st * 4 + 3, unpacked[0])
        self.assertEqual(_3rd * 4 + 3, unpacked[1])

    def _test_issue_146_group_packed_12(self):
        _1st = 0xff
        _3rd = 0xff
        ba = bytes([_1st, 0xff, _3rd])
        packed = np.frombuffer(ba, dtype=np.uint8)
        pf = Dictionary.get_proxy('BayerRG12Packed')
        unpacked = pf.expand(packed)
        self.assertEqual(_1st * 16 + 0xf, unpacked[0])
        self.assertEqual(_3rd * 16 + 0xf, unpacked[1])

    def _test_issue_146_packed_10(self):
        element = 0xff
        ba = bytes([element, element, element, element])
        packed = np.frombuffer(ba, dtype=np.uint8)
        pf = Dictionary.get_proxy('Mono10p')
        unpacked = pf.expand(packed)
        self.assertEqual(0x3ff, unpacked[0])
        self.assertEqual(0x3ff, unpacked[1])
        self.assertEqual(0x7ff, unpacked[2])

    def _test_issue_146_packed_12(self):
        _1st = 0xff
        _3rd = 0xff
        ba = bytes([_1st, 0xff, _3rd])
        packed = np.frombuffer(ba, dtype=np.uint8)
        pf = Dictionary.get_proxy('Mono12p')
        unpacked = pf.expand(packed)
        self.assertEqual(0xf * 256 + _1st, unpacked[0])
        self.assertEqual(_3rd * 16 + 0xf, unpacked[1])

    def _test_issue_146_mono_unpacked_multibytes(self):
        names = ['Mono10', 'Mono12']
        maximums = [0x4, 0x10]
        for index, name in enumerate(names):
            pf = Dictionary.get_proxy(name)
            for i in range(maximums[index]):
                for j in range(0x100):
                    ba = bytes([j, i, j, i])
                    packed = np.frombuffer(ba, dtype=np.uint8)
                    unpacked = pf.expand(packed)
                    # We know it's redundant but check 10 times:
                    for l in range(10):
                        for k in range(2):
                            self.assertEqual((i << 8) + j, unpacked[k])

class _TestIssue81(threading.Thread):
    def __init__(self, message_queue=None, cti_file_path=None):
        super().__init__()
        self._message_queue = message_queue
        self._cti_file_path = cti_file_path

    def run(self):
        h = Harvester()
        h.add_file(self._cti_file_path)
        h.update()
        try:
            ia = h.create_image_acquirer(0)
        except:
            # Transfer the exception anyway:
            self._message_queue.put(sys.exc_info())
        else:
            ia.start_acquisition()
            ia.stop_acquisition()
            ia.destroy()
            h.reset()

class TestIssue81(unittest.TestCase):
    _cti_file_path = get_cti_file_path()
    sys.path.append(_cti_file_path)

    def test_issue_81(self):
        message_queue = Queue()
        t = _TestIssue81(
            message_queue=message_queue, cti_file_path=self._cti_file_path
        )
        t.start()
        t.join()
        try:
            result = message_queue.get(block=False)
        except Empty:
            # Nothing happened; everything is fine.
            pass
        else:
            exception, message, backtrace = result
            # Transfer the exception:
            raise exception(message)

class TestIssue85(unittest.TestCase):
    _cti_file_path = get_cti_file_path()
    sys.path.append(_cti_file_path)

    def setUp(self) -> None:
        #
        self.env_var = 'HARVESTERS_XML_FILE_DIR'
        self.original = None if os.environ else os.environ[self.env_var]

    def tearDown(self) -> None:
        if self.original:
            os.environ[self.env_var] = self.original

    def test_issue_85(self):
        #
        temp_dir = os.path.join(
            gettempdir(), 'harvester', self.test_issue_85.__name__
        )

        #
        if os.path.isdir(temp_dir):
            rmtree(temp_dir)
        os.makedirs(temp_dir)

        #
        os.environ[self.env_var] = temp_dir

        #
        self.assertFalse(os.listdir(temp_dir))

        #
        with Harvester() as h:
            h.add_file(self._cti_file_path)
            h.update()
            with h.create_image_acquirer(0):
                # Check if XML files have been stored in the expected
                # directory:
                self.assertTrue(os.listdir(temp_dir))

class _OnNewBufferAvailable(Callback):
    def __init__(self, ia: ImageAcquirer, buffers: list):
        super().__init__()
        self._ia = ia
        self._buffers = buffers

    def emit(self, context: Optional[object] = None) -> None:
        buffer = self._ia.fetch_buffer()
        self._buffers.append(buffer)

    @property
    def buffers(self):
        return self._buffers

class _OnReturnBufferNow(Callback):
    def __init__(self, holder: _OnNewBufferAvailable):
        super().__init__()
        self._holder = holder

    def emit(self, context: Optional[object] = None) -> None:
        # Return/Queue the buffers before stopping image acquisition:
        while len(self._holder.buffers) > 0:
            buffer = self._holder.buffers.pop(-1)
            buffer.queue()

class TestIssue181(unittest.TestCase):
    def test_issue_181_with_nonexistent_file(self):
        h = Harvester()
        with self.assertRaises(FileNotFoundError):
            h.add_file('just a string', check_existence=True)

    def test_issue_181_with_invalid_file(self):
        h = Harvester()
        with self.assertRaises(OSError):
            h.add_file(__file__, check_validity=True)

class TestIssue188(unittest.TestCase):
    _height = 1
    _range = range(0, 3, 1)

    def test_issue_188_unpacked(self):
        proxies = [Mono8, Mono10, Mono12, Mono14, Mono16]
        expected_results = [
            [1, 2, 2, 2, 2],  # 1 x 1
            [2, 4, 4, 4, 4],  # 2 x 1
            [3, 6, 6, 6, 6],  # 3 x 1
        ]
        for i in self._range:
            for j, proxy in enumerate(proxies):
                self.assertEqual(
                    expected_results[i][j],
                    Component2DImage._get_nr_bytes(
                        pf_proxy=proxy(), width=i + 1, height=self._height
                    )
                )

    def test_issue_188_packed(self):
        proxies = [Mono10Packed, Mono12Packed]
        expected_results = [
            [2, 2],  # 1 x 1
            [3, 3],  # 2 x 1
            [4, 5],  # 3 x 1
        ]
        for i in self._range:
            for j, proxy in enumerate(proxies):
                self.assertEqual(
                    expected_results[i][j],
                    Component2DImage._get_nr_bytes(
                        pf_proxy=proxy(), width=i + 1, height=self._height
                    )
                )

    def test_issue_188_p(self):
        proxies = [Mono10p, Mono12p, Mono14p]
        expected_results = [
            [2, 2, 2],  # 1 x 1
            [3, 3, 4],  # 2 x 1
            [4, 5, 6],  # 3 x 1
        ]
        for i in self._range:
            for j, proxy in enumerate(proxies):
                self.assertEqual(
                    expected_results[i][j],
                    Component2DImage._get_nr_bytes(
                        pf_proxy=proxy(), width=i + 1, height=self._height
                    )
                )

    def test_issue_188_neels_case(self):
        proxies = [Mono12]
        width = 2456
        height = 2058
        for proxy in proxies:
            self.assertEqual(
                10108896,  # = 2456 * 2058 * 2
                Component2DImage._get_nr_bytes(
                    pf_proxy=proxy(), width=width, height=height
                )
            )

if __name__ == '__main__':
    unittest.main()


import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
from config.config import config
torch.set_printoptions(profile="full")

class SSTLoss(nn.Module):
    def __init__(self, use_gpu=config['cuda']):
        super(SSTLoss, self).__init__()
        self.use_gpu = use_gpu
        self.max_object = config['max_object']
        self.false_objects_column = None
        self.false_objects_row = None
        self.false_constant = config['false_constant']

    def add_unmatched_dim(self, x):
        if self.false_objects_column is not None and self.false_objects_column.shape[0]!=x.shape[0]:
            self.false_objects_column = None
            self.false_objects_row = None
        if self.false_objects_column is None:
            self.false_objects_column = Variable(
                torch.ones(x.shape[0], x.shape[1], x.shape[2], 1)) * self.false_constant
            if self.use_gpu:
                self.false_objects_column = self.false_objects_column.cuda()
        x = torch.cat([x, self.false_objects_column], 3)

        if self.false_objects_row is None:
            self.false_objects_row = Variable(torch.ones(x.shape[0], x.shape[1], 1, x.shape[3])) * self.false_constant
            if self.use_gpu:
                self.false_objects_row = self.false_objects_row.cuda()
        x = torch.cat([x, self.false_objects_row], 2)
        return x

    def euclidean_dist(self, x, y):
        square_x = x.pow(2)
        square_x = square_x.sum(-1)
        square_y = y.pow(2)
        square_y = square_y.sum(-1)
        ex = square_x.unsqueeze(-1)
        ey = square_y.unsqueeze(-2)
        xy = torch.einsum('bij,bkj->bik', x, y)
        exy = ex - 2 * xy + ey + 1e-10
        dist = exy.sqrt()
        if self.use_gpu:
            dist = dist.unsqueeze(1).cuda()
        else:
            dist = dist.unsqueeze(1)
        dist = self.add_unmatched_dim(dist)
        return dist

    def forward(self, current_feature, next_feature, target, mask0, mask1):
        input = self.euclidean_dist(current_feature, next_feature)
        input = 1 / (input + 0.01)
        mask_pre = mask0[:, :, :]
        mask_next = mask1[:, :, :]

        # TODO: 是否要修改mask的大小？ batch_size 问题需要修改
        mask0 = mask0.unsqueeze(3).repeat(1, 1, 1, self.max_object + 1)
        mask1 = mask1.unsqueeze(2).repeat(1, 1, self.max_object + 1, 1)
        mask0 = Variable(mask0.data)
        mask1 = Variable(mask1.data)
        target = Variable(target.byte().data)

        if self.use_gpu:
            mask0 = mask0.cuda()
            mask1 = mask1.cuda()

        mask_region = (mask0 * mask1).float()  # the valid position mask
        mask_region_pre = mask_region.clone()  # note: should use clone (fix this bug)
        mask_region_pre[:, :, self.max_object, :] = 0
        mask_region_next = mask_region.clone()  # note: should use clone (fix this bug)
        mask_region_next[:, :, :, self.max_object] = 0
        mask_region_union = mask_region_pre * mask_region_next

        input_pre = nn.Softmax(dim=3)(mask_region_pre * input)
        input_next = nn.Softmax(dim=2)(mask_region_next * input)
        input_all = input_pre.clone()
        input_all[:, :, :self.max_object, :self.max_object] = torch.max(input_pre, input_next)[:, :, :self.max_object,
                                                              :self.max_object]
        # input_all[:, :, :self.max_object, :self.max_object] = ((input_pre + input_next)/2.0)[:, :,
        # :self.max_object, :self.max_object]
        target = target.float()
        target_pre = mask_region_pre * target
        target_next = mask_region_next * target
        target_union = mask_region_union * target
        target_num = target.sum()
        target_num_pre = target_pre.sum()
        target_num_next = target_next.sum()
        target_num_union = target_union.sum()
        # todo: remove the last row negative effect
        if int(target_num_pre.item()):
            loss_pre = - (target_pre * torch.log(input_pre)).sum() / target_num_pre
        else:
            loss_pre = - (target_pre * torch.log(input_pre)).sum()
        if int(target_num_next.item()):
            loss_next = - (target_next * torch.log(input_next)).sum() / target_num_next
        else:
            loss_next = - (target_next * torch.log(input_next)).sum()
        if int(target_num_pre.item()) and int(target_num_next.item()):
            loss = -(target_pre * torch.log(input_all)).sum() / target_num_pre
        else:
            loss = -(target_pre * torch.log(input_all)).sum()

        if int(target_num_union.item()):
            loss_similarity = (target_union * (torch.abs((1 - input_pre) - (1 - input_next)))).sum() / target_num
        else:
            loss_similarity = (target_union * (torch.abs((1 - input_pre) - (1 - input_next)))).sum()

        _, indexes_ = target_pre.max(3)
        indexes_ = indexes_[:, :, :-1]
        _, indexes_pre = input_all.max(3)
        indexes_pre = indexes_pre[:, :, :-1]
        mask_pre_num = mask_pre[:, :, :-1].sum().item()
        print('111111111111111111111')
        if mask_pre_num:
            accuracy_pre = (indexes_pre[mask_pre[:, :, :-1]] == indexes_[
                mask_pre[:, :, :-1]]).float().sum() / mask_pre_num
        else:
            accuracy_pre = (indexes_pre[mask_pre[:, :, :-1]] == indexes_[mask_pre[:, :, :-1]]).float().sum() + 1
        print('222222222222222222222')

        _, indexes_ = target_next.max(2)
        indexes_ = indexes_[:, :, :-1]
        _, indexes_next = input_next.max(2)
        indexes_next = indexes_next[:, :, :-1]
        mask_next_num = mask_next[:, :, :-1].sum().item()
        if mask_next_num:
            print('333333333333333333333')
            accuracy_next = (indexes_next[mask_next[:, :, :-1]] == indexes_[
                mask_next[:, :, :-1]]).float().sum() / mask_next_num
            print('444444444444444444444')
        else:
            accuracy_next = (indexes_next[mask_next[:, :, :-1]] == indexes_[mask_next[:, :, :-1]]).float().sum() + 1
        return loss_pre, loss_next, loss_similarity, \
               (loss_pre + loss_next + loss + loss_similarity) / 4.0, accuracy_pre, accuracy_next, (
                       accuracy_pre + accuracy_next) / 2.0, indexes_pre

    def getProperty(self, input, target, mask0, mask1):
        return self.forward(input, target, mask0, mask1)


# -*- coding: utf-8 -*-
import time
import logging
import nerd_database
from nerd_module import wikipedia

class message_handler:
    '''
        ::message <TeleBot message>
        ::received_time <float> unix time
    '''
    def __init__(self, message, received_time):
        self.message = message
        if message.content_type == 'text': self.command = message.text.lower().strip()
        else: self.command = ''
        self.received_time = received_time
        self.cheat_code      = None
        self.cheat_cat       = None
        self.cheat_input     = None
        self.reserved_answer = None
        self.in_lod_id       = None
        self.in_log()
        self.out_log_done    = False

    def is_in_blacklist(self):
        if nerd_database.blacklist().find(self.command):
            return True
        return False

    def is_reserved_code(self):
        result = nerd_database.reserved_code().find(self.command)
        if result:
            self.reserved_answer = result
            return True
        return False

    '''
        Cheat Code: it's a reserved word that command starts with
            * without space, tab, newLine, or other space char
    '''
    def is_cheatcode(self):
        result = nerd_database.cheatcode().find(self.command)
        if result:
            self.cheat_cat   = result
            self.cheat_code  = self.command
            self.cheat_input = ''
            return True
        comm = self.command.split()[0].lower()
        result = nerd_database.cheatcode().find(comm)
        if result:
            self.cheat_cat   = result
            self.cheat_code  = comm
            self.cheat_input = self.command[len(comm)+1:].strip()
            return True
        return False

    '''
        Run cheat code
        ::tb <TeleBot> telegram_bot_handler
        @return
            * 0    No result
            * -1   Error
            * <str> string result or path of image
    '''
    def do_cheat_code(self, tb):
        if self.cheat_cat == 'food':          return self.cheat_code_food(tb)
        elif self.cheat_cat == 'wiki_en':     return self.do_wiki(tb, lang='en')
        elif self.cheat_cat == 'joke':        return self.cheat_code_joke(tb)
        else: return (-1, 'Cheat Code: No Function Defined!')

    def cheat_code_joke(self, tb):
        joke = nerd_database.joke().suggest()
        tb.send_chat_action(self.message.from_user.id, 'typing')
        tb.reply_to(self.message, joke)
        return (1, joke)

    def cheat_code_food(self, tb):
        food = nerd_database.iranian_foods().suggest()
        if food == 0 or food == None:    return (0, u'Cheat Code: %s' % self.cheat_cat)
        elif food == -1:                 return (-1, u'Cheat Code: %s' % self.cheat_cat)
        #there is a suggestion
        suggest = u'پیشنهاد من یک غذای ایرانی هست به اسم "%s"' % food
        wiki = wikipedia().summary(food)
        if wiki[0]: suggest += "\n\n" + wiki[1]
        tb.send_chat_action(self.message.from_user.id, 'typing')
        tb.reply_to(self.message, suggest)
        return (1, suggest)

    '''
        Search in wikipedia
        ::tb <TeleBot> telegram_bot_handler
        @return
            * 0    No result
            * -1   Error from wikipedia
            * -2   internet Error
            * <str> string result or path of image
    '''
    def do_wiki(self, tb, lang='fa'):
        if self.cheat_cat:  wiki = wikipedia().summary(self.cheat_input, lang=lang)
        else:               wiki = wikipedia().summary(self.command, lang='fa')
        if wiki[0] == 0:             return(0,  u'Wiki: %s' % wiki[1])
        elif wiki[0] in {-1, -2}:    return(-1, u'Wiki: %s' % wiki[1])
        else:
            tb.send_chat_action(self.message.from_user.id, 'typing')
            tb.reply_to(self.message, wiki[1])
            return(1, wiki[1])
        return(-1, 'Wiki: UnExpected Error')

    '''
        Update 'in' type log and return _id for save in 'out' type log and debugging
        @return <mongo _id>
    '''
    def in_log(self):
        user_fname =    self.message.from_user.first_name
        user_lname =    self.message.from_user.last_name
        user_id =       self.message.from_user.id
        user_username = self.message.from_user.username
        log_data = {'type':             'in',
                    'time':             self.received_time,
                    'user-fname':       user_fname,
                    'user-lname':       user_lname,
                    'user-id':          user_id,
                    'user-username':    user_username,
                    'detail':           self.command}
        #logging.info(log_data)
        logging.info(self.command)

    def out_log(self, ok_code, detail):
        if detail in {'', None}:         raise Exception('Detail is Empty')
        if type(ok_code) != type(int()): raise Exception('Error in "ok" code')
        log_data = {'type':   'out',
                    'ok':     ok_code,
                    'rel':    self.in_lod_id,
                    'detail': detail}
        log_data = detail
        self.out_log_done = True
        if ok_code == -1:
            logging.error(log_data)
        else:
            logging.info(log_data)

'''
    TeleBot thread run this function After i handle the messages
    ::messages <list> list of new messages
'''
def listener(messages, tb):
    ### Note:received_time must be before of for-loop for calculate response time
    receive_time = time.time()
    ###
    for m in messages:
        message = message_handler(m, receive_time)
        if message.command == '':
            message.out_log(0, 'Empty Command')
        elif message.is_in_blacklist():
            message.out_log(0, 'Black List')
        elif message.is_reserved_code():
            tb.send_chat_action(m.from_user.id, 'typing')
            tb.send_message(m.from_user.id, message.reserved_answer)
            message.out_log(1, message.reserved_answer)
        elif message.is_cheatcode():
            result = message.do_cheat_code(tb)
            message.out_log(result[0],  result[1])
        else:
            result = message.do_wiki(tb)
            message.out_log(result[0],  result[1])
        if not message.out_log_done:
            message.out_log(0, 'Magic Code!')


# pylint: disable=invalid-name, unnecessary-pass, consider-using-with

"""
    Setup for apache beam pipeline.
"""

from __future__ import absolute_import
from __future__ import print_function
import subprocess
from distutils.command.build import build as _build  # type: ignore
import setuptools

# This class handles the pip install mechanism.
class build(_build):  # pylint: disable=invalid-name
    """A build command class that will be invoked during package install.
    The package built using the current setup.py will be staged and later
    installed in the worker using `pip install package'. This class will be
    instantiated during install for this specific scenario and will trigger
    running the custom commands specified.
    """
    sub_commands = _build.sub_commands + [('CustomCommands', None)]

CUSTOM_COMMANDS = [
    ["gsutil","cp" ,"gs://gdw-shared-resources/certs/*" ,"/usr/local/share/ca-certificates/"],
    ["sudo","update-ca-certificates"],
    ["gsutil" ,"cp", "gs://gdw-shared-resources/init/pip.conf", "/etc/pip.conf"],
    ["sudo","apt-get","--assume-yes", "update"],
#     ["pip","install","apache-beam[gcp]"],
    ["/usr/local/bin/python3" ,"-m", "pip" ,"install" ,"--upgrade", "pip"],
    ["pip","install","python-snappy"],
    ["pip","install","google-cloud-storage"],
    ["pip","install","google-api-python-client"],
    ["pip","install","google-api-client"],
    ["pip","install","google-cloud-aiplatform"],
    ["pip","install","fsspec"],
    ["pip","install" ,"dill==0.3.2"],
    ["pip","install","tensorflow"],
    ["pip","install","tfx==0.29.0"],
    ["pip","install","kfp==1.6.1"],
    ["pip","install","gcsfs"]
                  ]

class CustomCommands(setuptools.Command):
    """A setuptools Command class able to run arbitrary commands."""
    def initialize_options(self):
        """Initialize options"""
        pass

    def finalize_options(self):
        """Finalize options"""
        pass

    def RunCustomCommand(self, command_list):   # pylint: disable=no-self-use
        """To run custom command"""
        print('Running command: %s' % command_list)
        p = subprocess.Popen(
        command_list,
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT)
    # Can use communicate(input='y\n'.encode()) if the command run requires
    # some confirmation.
        stdout_data, _ = p.communicate()
        print('Command output: %s' % stdout_data)
        if p.returncode != 0:
            raise RuntimeError(
          'Command %s failed: exit code: %s' % (command_list, p.returncode))

    def run(self):
        """Run function for custom commands"""
        for command in CUSTOM_COMMANDS:
            self.RunCustomCommand(command)

NAME = 'Unified Ai Online'
VERSION = '2.0'
REQUIRED_PACKAGES = [
    'apache-beam[gcp]',
    'tensorflow==2.3.0',
    'opencv-python',
    'gcsfs',
    'workflow'
    ]

setuptools.setup(
    name=NAME,
    version=VERSION,
    install_requires=[],
    packages=setuptools.find_packages(),
    cmdclass={
        # Command class instantiated and run during pip install scenarios.
        'build': build,
        'CustomCommands': CustomCommands
    }
)


# -*- coding=UTF-8 -*-
# pyright: strict

from __future__ import annotations

from typing import Optional, Tuple

import cv2
import numpy as np
from PIL.Image import Image

from ... import action, imagetools, mathtools, templates, app
from ..scene import Scene, SceneHolder
from ..unknown import UnknownScene

def _has_granted_reward(img: Image) -> bool:
    cv_img = imagetools.cv_image(imagetools.resize(img, height=25).convert("L"))
    blur_img = cv2.GaussianBlur(cv_img, (5, 9), 1)
    diff_img = blur_img - cv_img
    diff_avg = np.average(diff_img)
    ret = diff_avg > 100
    app.log.image(
        "has granted reward: %s diff_avg=%s" % (ret, diff_avg),
        cv_img,
        level=app.DEBUG,
        layers={
            "blur": blur_img,
            "diff": diff_img,
        },
    )

    return ret

def _iter_reward_bbox(rp: mathtools.ResizeProxy):
    for t in (330, 533, 735):
        yield rp.vector4((400, t, 500, t + 25), 540)

class CompetitorMenuScene(Scene):
    @classmethod
    def name(cls):
        return "paddock"

    @classmethod
    def _enter(cls, ctx: SceneHolder) -> Scene:
        action.wait_image_stable(templates.TEAM_RACE_CHOOSE_COMPETITOR)
        return cls()

    def locate_granted_reward(self) -> Optional[Tuple[int, int]]:
        rp = action.resize_proxy()
        screenshot = app.device.screenshot()
        for bbox in _iter_reward_bbox(rp):
            if _has_granted_reward(screenshot.crop(bbox)):
                return (bbox[0], bbox[1])

    def choose(self, ctx: SceneHolder, index: int) -> None:
        """choose competitor by index, topmost option is 0."""
        rp = action.resize_proxy()
        for i, bbox in enumerate(_iter_reward_bbox(rp)):
            if i == index:
                app.device.tap(
                    (
                        bbox[0],
                        bbox[1],
                        30,
                        30,
                    )
                )
                UnknownScene.enter(ctx)
                return
        raise IndexError(index)