import os
import re

import pytest

from pycondor.job import Job
from pycondor.dagman import Dagman
from pycondor.visualize import visualize, extract_format, dag_to_graphviz
from pycondor.utils import clear_pycondor_environment_variables

graphviz = pytest.importorskip('graphviz')  # noqa: E402

clear_pycondor_environment_variables()

here = os.path.abspath(os.path.dirname(__file__))
example_script = os.path.join(here, 'example_script.py')

@pytest.fixture()
def dagman(tmpdir_factory):
    dag = Dagman(name='example_dagman')

    merge = Job(name='merge',
                executable='merge.py',
                dag=dag)

    for i in range(5):
        processing = Job(name='processing_{}'.format(i),
                         executable='process.py',
                         dag=dag)
        merge.add_parent(processing)

    cleanup_dag = Dagman(name='cleanup',
                         dag=dag)
    cleanup_dag.add_parent(merge)
    return dag

def test_visualize_save_file(dagman, tmpdir):
    filename = str(tmpdir.join('viz.png'))
    visualize(dagman, filename)
    assert os.path.exists(filename)

@pytest.mark.parametrize('filename, expected', [
    ('myfile.png', 'png'),
    ('myfile.pdf', 'pdf'),
    ('myfile.dot', 'dot'),
    ('myfile.svg', 'svg'),
    ('myfile.jpeg', 'jpeg'),
    ('myfile.jpg', 'jpg'),
])
def test_extract_format(filename, expected):
    assert extract_format(filename) == expected

def test_extract_format_invalid_format(dagman):
    with pytest.raises(ValueError) as excinfo:
        extract_format('dag_graph.csv')
    assert 'invalid format' in str(excinfo.value).lower()

def test_dag_to_graphviz(dagman):
    g = dag_to_graphviz(dagman)
    assert isinstance(g, graphviz.Digraph)

def test_graph_shapes(dagman):
    g = dag_to_graphviz(dagman)

    shapes = {}
    label_shape_re = re.compile(r'.*\[label=(.*?) shape=(.*?)\]')
    for line in g.body:
        match = label_shape_re.match(line)
        if match:
            name = match.group(1)
            shape = match.group(2)
            shapes[name] = shape
        else:
            continue

    # Check node names
    assert set(node.name for node in dagman) == set(shapes.keys())
    # Check node shapes
    for node in dagman:
        expected_shape = 'circle' if isinstance(node, Job) else 'square'
        assert shapes[node.name] == expected_shape

def test_visualize_method(dagman):
    graph_vis_func = visualize(dagman).body
    graph_vis_method = dagman.visualize().body
    assert graph_vis_func == graph_vis_method


import sys
from bs4 import BeautifulSoup

# Read the HTML document from stdin
html_doc = sys.stdin.read()

# Create a BeautifulSoup object
soup = BeautifulSoup(html_doc, 'html.parser')

for tag in soup.find_all(True):
    # Remove all inline styles
    if tag.has_attr('style'):
        del tag['style']

    # Remove ids (we can add these back later if needed)
    if tag.has_attr('id'):
        del tag['id']

    # Remove classes
    if tag.has_attr('class'):
        del tag['class']

    # Remove colspan / rowspan if they're set to the default (1)
    if tag.get('colspan') == "1":
        del tag['colspan']
    if tag.get('rowspan') == "1":
        del tag['rowspan']

# Remove unnecessary span tags
for span in soup.find_all('span'):
    span.unwrap()

# Remove empty paragraphs
for p in soup.find_all('p'):
    if not p.contents:
        p.extract()

# Remove empty anchors
for p in soup.find_all('a'):
    if not p.contents:
        p.extract()

# Pretty print the modified HTML
print(soup.prettify())


import numpy as np
import os
import cv2
import time
import colorama

PIXEL = '▀'

def create_pixel(rgb1,rgb2, is_last=False):
    if is_last:
         return f'\x1b[38;2;{rgb1[0]};{rgb1[1]};{rgb1[2]}m' + PIXEL
    else:
        return f'\x1b[38;2;{rgb1[0]};{rgb1[1]};{rgb1[2]}m' + f'\x1b[48;2;{rgb2[0]};{rgb2[1]};{rgb2[2]}m' + PIXEL
def print_screen(image, rewrite=False):
    size = os.get_terminal_size()

    x = size[0] - 1
    y = (size[1]*2) -8
    if (image.shape[0]/x >=1 or  image.shape[1]/y>=1):
        if image.shape[0]/x >=  image.shape[1]/y:
            resize = x/image.shape[0]
        else:
            resize = y/image.shape[1]

        image = cv2.resize(image, (int(image.shape[1]*resize), int(image.shape[0]*resize)), interpolation = cv2.INTER_AREA)
    string_image = ''
    if rewrite:
        old_height = image.shape[0]//2
    for i in range(image.shape[0]//2):
        string_row = ''
        row1 = image[i*2]
        row2 = image[(i*2)+1]
        for pix1, pix2 in zip(row1,row2):
            string_row = string_row + create_pixel(pix1,pix2)
        string_image = string_image + '\n' +string_row + '\x1b[0m'
    if len(image) % 2:
        string_row = [create_pixel(pix, (0,0,0), is_last=True) for pix in image[-1]]
        string_row = ''.join(string_row)
        string_image = string_image + '\n' +string_row + '\x1b[0m'
        if rewrite:

            old_height = old_height + 1
    print(string_image, end='')
    if rewrite:
        print((old_height)*'\033[A', end ='')

def print_image(path='pizza2.png'):
    image = cv2.imread(path, cv2.IMREAD_UNCHANGED)
    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)

    print_screen(image)
def print_animated(path, loop = True, duration = 0.1, do_clear = False):

    cap = cv2.VideoCapture(path)

    while True:
        old_time = time.time()
        ret, frame = cap.read()
        if not ret:
            if loop:
                cap.set(cv2.CAP_PROP_POS_FRAMES, 0)
                ret, frame = cap.read()
            else:
                break
        if cv2.waitKey(25) & 0xFF == ord('q'):
            print('Bye')
            break
        frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
        if do_clear:
            os.system('cls' if os.name == 'nt' else 'clear')
        print_screen(frame, rewrite=True)
        new_time = time.time()
        if duration-(new_time-old_time)> 0 : time.sleep(duration-(new_time-old_time))

def print_camera(number=0, do_clear=False):
    cap = cv2.VideoCapture(0)
    if not cap.isOpened():
        print("Cannot open camera")
        exit()
    while True:
        ret, frame = cap.read()
        if cv2.waitKey(25) & 0xFF == ord('q'):
            break

        frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
        if do_clear:
            os.system('cls' if os.name == 'nt' else 'clear')
        print_screen(frame, rewrite=True)
#array_image = np.asarray( image, dtype="int32" )
#print_image('pizza2.png')
#print_animated('rickroll-rick.gif')
print_camera()


import pandas as pd
import geopandas as gpd
from geopandas import GeoDataFrame
import numpy as np

import rasterio
from rasterio.crs import CRS
import rasterio.mask
import rasterio.plot

from scipy.ndimage import distance_transform_edt
from affine import Affine
import matplotlib.pyplot as plt

from shapely.geometry import box
from shapely.geometry import LineString
from shapely.geometry import Point

from pathlib import Path
from typing import Tuple
from typing import Any

# Generate expanded bounding box for rasterization
def expand_bounding_box(gdf: GeoDataFrame) -> pd.DataFrame:

    # Generate original bounding box
    bbox = gdf.geometry.bounds

    # Extract bounding box coordinates as floats using iloc[0]
    minx = float(bbox.minx.iloc[0])
    miny = float(bbox.miny.iloc[0])
    maxx = float(bbox.maxx.iloc[0])
    maxy = float(bbox.maxy.iloc[0])

    # Calculate the expansion sizes for height and width
    height_expansion = (maxy - miny)
    width_expansion = (maxx - minx)

    # Create new bounding coordinates

    new_minx = minx - width_expansion  # Expanding left
    new_miny = miny - height_expansion  # Expanding bottom
    new_maxx = maxx + width_expansion  # Expanding right
    new_maxy = maxy + height_expansion  # Expanding top

    expanded_bbox = pd.DataFrame({
        'minx': [new_minx],
        'miny': [new_miny],
        'maxx': [new_maxx],
        'maxy': [new_maxy]
    })

    return expanded_bbox

# Get Meta data from TIF file
def get_metadata(tif_file_path: Path) -> Tuple[dict, Tuple[int, int], Affine, CRS]:
    with rasterio.open(tif_file_path) as src:
        out_meta = src.meta
        shape = src.shape

    # Transform Data
    transform = out_meta['transform']
    # Out metadata is exactly the same as our in metadata.
    target_crs = out_meta['crs']

    return out_meta, shape, transform, target_crs

# Generate expanded shape and new transform
def expand_shape_and_transform(original_shape: Tuple[int, int], original_transform: Affine) -> Tuple[Tuple[int, int], Affine]:

    # New shape
    new_shape = (original_shape[0]*3, original_shape[1]*3)

    # New transform
    # Calculate the adjustment considering the resolution
    height = original_shape[0]
    width = original_shape[1]

    resolution = original_transform.a
    adjustment_x = resolution * width  # Width adjustment (to the left)
    adjustment_y = resolution * height  # Height adjustment (up)

    # Update the transform by adding adjustments
    new_transform = Affine(original_transform.a, original_transform.b, original_transform.c - adjustment_x,
                        original_transform.d, original_transform.e, original_transform.f + adjustment_y)

    return new_shape, new_transform

# Rasterize geometeries found in a GDF onto an empty array based on an existing transform
def rasterize(gdf: GeoDataFrame, out_shape: Tuple[int, int], transform: Affine) -> np.ndarray:
    # Rasterize the GeoDataFrame
    rasterized = rasterio.features.rasterize(
        [(geom, 1) for geom in gdf.geometry],
        out_shape=out_shape,
        transform=transform,
        fill=0,
        all_touched=True,
        dtype=rasterio.uint8
    )

    # Switch 1s and 0s
    rasterized = np.logical_not(rasterized).astype(int)
    return rasterized

# Subset expanded distance array to original size
def subset_array(original_shape: Tuple[int, int], computed_array: np.ndarray) -> np.ndarray:

    # Create rows and columns
    start_row = original_shape[0]
    end_row = original_shape[0]*2
    start_col = original_shape[1]
    end_col = original_shape[1]*2

    # Subset original array from expanded distance array
    original_out = computed_array[start_row:end_row, start_col:end_col]

    return original_out

# Used to stamp a country border on an existing raster image
def mask_admin(in_path: Path, admin_path: Path, out_path: Path) -> None:
    with rasterio.open(in_path) as src:
        admin = gpd.read_file(admin_path).to_crs(src.crs)
        kwargs = {
            'all_touched': True,
            'nodata': np.nan,
        }
        out_image, out_transform = rasterio.mask.mask(
            src,
            admin.geometry.tolist(),
            **kwargs
        )
        out_meta = src.meta
    out_meta.update({"driver": "GTiff",
                     "nodata": np.nan,
                     "height": out_image.shape[1],
                     "width": out_image.shape[2],
                     "transform": out_transform})

    with rasterio.open(out_path, "w", **out_meta) as dest:
        dest.write(out_image)

# Create Diagnostic Plots
def generate_plots(admin_gdf: GeoDataFrame, vector_gdf: GeoDataFrame, out_path: Path, save_path: Path) -> None:
    fig, axs = plt.subplots(1, 3, figsize=(30, 10))
    # Read in New Raster
    with rasterio.open(out_path) as src:
        combined_raster = src.read(1, masked=True)
        transform = src.transform
    log_combined_raster = np.log(1 + combined_raster)
    # Plot 1: Original vector geometries and admin boundary
    vector_gdf.plot(ax=axs[0], edgecolor='red', facecolor='none')
    admin_gdf.boundary.plot(ax=axs[0], color='k')
    axs[0].set_title('Original Vector Geometries and Admin Boundary')
    # Plot 2: Combined raster with admin boundary overlay
    im = axs[1].imshow(combined_raster, extent=(transform[2], transform[2] + transform[0]*combined_raster.shape[1],
                                       transform[5] + transform[4]*combined_raster.shape[0], transform[5]), cmap='viridis')
    fig.colorbar(im, ax=axs[1])
    axs[1].set_title('Distance with Country Overlay')
    admin_gdf.boundary.plot(ax=axs[1], color='red')
    # Plot 3: Log-transformed raster with admin boundary overlay
    im = axs[2].imshow(log_combined_raster, extent=(transform[2], transform[2] + transform[0]*combined_raster.shape[1],
                                       transform[5] + transform[4]*combined_raster.shape[0], transform[5]), cmap='viridis')
    fig.colorbar(im, ax=axs[2])
    axs[2].set_title('Log Distance with Country Overlay')
    admin_gdf.boundary.plot(ax=axs[2], color='red')

    plt.tight_layout()
    plt.savefig(save_path)

    return

# Main function that takes in path files and processes through all the steps to compute distance
def compute_distance(admin0_path: Path, vector_file_path: Path, meta_data_path: Path, out_path: Path, image_save_path: Path) -> None:
    # Step 1:Read in admin0 file
    admin0_path = admin0_path
    admin0 = gpd.read_file(admin0_path)
    # Step 2:Generate Expanded Bounding Box
    expanded_bbox = expand_bounding_box(admin0)
    # Step 3:Read in vector file
    vector_file_path = vector_file_path
    vector_gdf = gpd.read_file(vector_file_path)
    vector_gdf = vector_gdf.to_crs(admin0.crs)
    # Step 4:Subest Vector file path with Expanded Bounding Box
    vector_gdf_subset = vector_gdf.cx[expanded_bbox.iloc[0]['minx']:expanded_bbox.iloc[0]['maxx'],
                              expanded_bbox.iloc[0]['miny']:expanded_bbox.iloc[0]['maxy']]
    # Step 5:Read in TIF file for metadata
    out_meta, shape, transform, target_crs = get_metadata(meta_data_path)
    # Step 6:Generate new transform and shape
    new_shape, new_transform = expand_shape_and_transform(shape, transform)
    # Step 7:Align files to target_crs
    admin0 = admin0.to_crs(target_crs)
    vector_gdf_subset = vector_gdf_subset.to_crs(target_crs)
    # Step 8:Rasterize vector file
    rasterized = rasterize(vector_gdf_subset, new_shape, new_transform)
    # Step 9:Compute Distance
    out = distance_transform_edt(rasterized)
    print("Distance computed")
    # Step 10:Subset original array shape from computed distances
    original_out = subset_array(shape, out)
    # Step 11:Write out tif file
    original_out = original_out.astype(out_meta['dtype'])
    with rasterio.open(out_path, "w", **out_meta) as dest:
        dest.write(original_out.reshape((1, *original_out.shape)))
    # Step 12:Stamp out border
    mask_admin(out_path, admin0_path, out_path)
    # Step 13:Create Diagnostic Plots
    generate_plots(admin0, vector_gdf_subset, out_path, image_save_path)
    print('TIF saved at', out_path)
    print('Plots saved at', image_save_path)
    return


import csv
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split

class Knn():
    def __init__(self):
        # self.csvfile = open('test.csv', 'r')
        # self.bdd_apprentissage = csv.DictReader(self.csvfile, delimiter=',')
        pass

    def distance_tweet(self, tweet_a, tweet_b):
        list_words_a = tweet_a.split(" ")
        list_words_b = tweet_b.split(" ")
        nb_common_words = 0
        nb_max_words = max(len(list_words_a), len(list_words_b))

        for word_a in list_words_a:
            if word_a in list_words_b:
                nb_common_words += 1
        return (nb_max_words - nb_common_words) / nb_max_words

    def rate_tweet(self, k, tweet):
        df = pd.read_csv('bdd.csv')
        t = [tweet]
        # t = np.array([tweet])
        df['distance'] = df['tweet'].apply(self.distance_tweet, args=t)
        k_voisins = np.array(df.sort_values('distance')[:k]['note'], int)
        k_voisins[k_voisins == -1] = 10 # car bincount n'accepte pas les valeurs négatives
        counts = np.bincount(k_voisins)
        note = np.argmax(counts)

        if note == 10:
           return -1
        else:
           return note

    def count_occurences(self, neighbours):
        nb_occurence = {"positives": [0,1], "negatives": [0,-1], "neutres":[0,0]}
        for (_, note) in neighbours:
            if note > 0:
                nb_occurence["positives"][0] += 1
            elif note == 0:
                nb_occurence["neutres"][0] += 1
            else :
                nb_occurence["negatives"][0] += 1
        return nb_occurence

    def calcul_classe(self, target_class, predict_class):
        df = pd.read_csv('bdd.csv')
        df['note_1nn'] = df['tweet'].apply(self.rate_tweet, args=1)

        dfn = df[df['note'] == target_class]
        return dfn[dfn['note_1nn'] == predict_class].shape[0]

    def test(self, k):
        df = pd.read_csv('bdd.csv')

        X = df['tweet']
        y = df['note']
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)

        predict = []

        for index, value in X_test.items():
            predict.append(self.rate_tweet(k, value))

        y_test_val = list(y_test)

        v1 = self.calcul_classe(y_test_val, predict, 1, 1)
        v2 = self.calcul_classe(y_test_val, predict, 1, -1)
        v3 = self.calcul_classe(y_test_val, predict, 1, 0)
        v4 = self.calcul_classe(y_test_val, predict, -1, 1)
        v5 = self.calcul_classe(y_test_val, predict, -1, -1)
        v6 = self.calcul_classe(y_test_val, predict, -1, 0)
        v7 = self.calcul_classe(y_test_val, predict, 0, 1)
        v8 = self.calcul_classe(y_test_val, predict, 0, -1)
        v9 = self.calcul_classe(y_test_val, predict, 0, 0)

        print(v1, v2, v3, v4, v5, v6, v7, v8, v9)

    def calcul_classe(self, y_val, predict_val, target_class, predict_class):
        comp = 0
        for i in range(len(y_val)):
            if y_val[i] == target_class and predict_val[i] == predict_class:
                comp += 1
        return comp

        dfn = df[df['note'] == target_class]
        return dfn[dfn['note_dico'] == predict_class].shape[0]

# knn = Knn()
# print(knn.rate_tweet(1, "Ce n'est pas la meilleure équipe de l'histoire."))
# print(knn.calcul_classe(1, 1))
# knn.test(3)


from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.model_selection import train_test_split
import pandas as pd
import numpy as np
import json

#clase necesaria para la codificacion de numpy a json
class NumpyEncoder(json.JSONEncoder):
    """ Custom encoder for numpy data types """
    def default(self, obj):
        if isinstance(obj, (np.int_, np.intc, np.intp, np.int8,np.int16, np.int32, np.int64, np.uint8,np.uint16, np.uint32, np.uint64)):
            return int(obj)

        elif isinstance(obj, (np.float_, np.float16, np.float32, np.float64)):
            return float(obj)

        elif isinstance(obj, (np.complex_, np.complex64, np.complex128)):
            return {'real': obj.real, 'imag': obj.imag}

        elif isinstance(obj, (np.ndarray,)):
            return obj.tolist()

        elif isinstance(obj, (np.bool_)):
            return bool(obj)

        elif isinstance(obj, (np.void)):
            return None

        return json.JSONEncoder.default(self, obj)

#construir el vectorizador
#countvec = CountVectorizer(ngram_range=(1,1), stop_words="english", encoding="utf-8")
countvec = CountVectorizer()

#varialbe general compartida del vectorizador de genero

def buildJsonEntireData(data):

    #datos textuales
    colGenero = data["genre"]
    featuresGenero = countvec.fit_transform(colGenero)
    vectGeneroTotal = featuresGenero.toarray()
    print("lenGenero:",featuresGenero.shape)

    #datos numericos
    colNumerics = data[["Duration","year","rating"]]
    dataTotal = data.to_json(orient="records")
    parsedObjTotal = json.loads(dataTotal)

    #formando el json data
    datajson = {"data":[]}

    #recorremos cada registro de data y seleccionar especificas columnas
    for x in range(len(data)):
        numericData = [ data["Duration"][x], data["year"][x], data["rating"][x] ]
        datajson["data"].append({
            "vector": numericData + featuresGenero.toarray()[x].tolist(),
            "obj": parsedObjTotal[x]
        })

    #usando la clase para tenerlo como un array numpy
    json_data = json.dumps(datajson, cls=NumpyEncoder)
    #print(json_data)

    #transformando a un archivo json del numpy array
    with open("json/totalmovies.json", "w") as outfile:
        outfile.write(json_data)

    return vectGeneroTotal, json_data

def buildJsonSelectedMovie(vectGenTotal, selectMovie):

    #conocer el index del dataframe
    indexDfMovie = selectMovie.index[0]

    #datos textuales
    colGenero = selectMovie["genre"]
    featuresGenero = countvec.fit_transform(colGenero)

    dataMovie = selectMovie.to_json(orient="records")
    parsedObjMovie = json.loads(dataMovie)

    #formando el json data
    datajson = {"data":[]}

    #recorremos cada registro de data y seleccionar especificas columnas
    numericData = [selectMovie.loc[indexDfMovie]["Duration"], selectMovie.loc[indexDfMovie]["year"], selectMovie.loc[indexDfMovie]["rating"]]

    datajson["data"].append({
        "vector": numericData + vectGenTotal[indexDfMovie].tolist(),
        "obj": parsedObjMovie[0]
    })

    #usando la clase para tenerlo como un array numpy
    json_data = json.dumps(datajson, cls=NumpyEncoder)
    #print(json_data)

    #transformando a un archivo json del numpy array
    with open("json/selectMovie.json", "w") as outfile:
        outfile.write(json_data)

    return json_data


from django.contrib import admin
from .models import User, AuctionBids, AuctionListing,AuctionListingComments,Watchlist
# Register your models here.

class AuctionAdminForm(admin.ModelAdmin):
    def save_model(self, request, obj, form, change):
        if not obj.creator:
            obj.creator = request.user  # Ustaw właściciela aukcji na aktualnie zalogowanego użytkownika
        super().save_model(request, obj, form, change)

admin.site.register(User)
admin.site.register(AuctionBids)
admin.site.register(AuctionListingComments)
admin.site.register(AuctionListing, AuctionAdminForm)
admin.site.register(Watchlist)


import argparse
import datetime
import io
import os
import phpserialize
import tarfile

from pathlib import Path

def filter_changes_file(path):
  output = io.BytesIO()
  with open(path, 'rb') as infile:
    for ln in infile:
      fields = ln.split(b'\t', 2)
      output.write(b'\t'.join([fields[0], b'redacted-ip', fields[2]]))
  size = output.tell()
  output.seek(0)
  return output, size

def filter_meta_file(path):
  with open(path, 'rb') as f:
    data = phpserialize.load(f)

  for key in [b'current', b'persistent']:
    if key in data and b'last_change' in data[key] and isinstance(data[key][b'last_change'], dict) and b'ip' in data[key][b'last_change']:
      if data[key][b'last_change'][b'ip'] != b'127.0.0.1':
        data[key][b'last_change'][b'ip'] = b'redacted-ip'

  output = io.BytesIO()
  phpserialize.dump(data, output)
  size = output.tell()
  output.seek(0)

  return output, size

def make_empty_users_auth_file():
  """
  Return a default dokuwiki conf/users.auth.php file (with no users).
  """
  output = io.BytesIO()
  output.write(b'''# users.auth.php
# <?php exit()?>
# Don't modify the lines above
#
# Userfile
#
# Format:
#
# login:passwordhash:Real Name:email:groups,comma,separated
''')
  size = output.tell()
  output.seek(0)
  return output, size

class DokuwikiBackuper:
  def __init__(self):
    self.arg_parser = argparse.ArgumentParser(allow_abbrev=False)
    self.arg_parser.add_argument('--output-dir', '-o', help='Directory where the backup archive should be stored. It is passed through strftime before it is used. Created if it does not exist.', required=True)
    self.arg_parser.add_argument('--name-prefix', '-p', type=str, default=(), nargs=1, help='Optional prefix to prepend to the output archive name.')
    self.arg_parser.add_argument('--keep-users', action='store_false', dest='exclude_users', help='Include the user file in the backup. The default is to exclude it.')
    self.arg_parser.add_argument('--keep-ips', action='store_false', dest='strip_ips', help='Keep IP addresses in the backup. The default is to strip them.')
    self.arg_parser.add_argument('dokuwiki_root', type=Path, help='Path to the dokuwiki directory that should be backed up.')

  def run(self):
    self.args = self.arg_parser.parse_args()

    now = datetime.datetime.utcnow()

    output_dir = Path(now.strftime(self.args.output_dir))

    output_filename = '-'.join(list(self.args.name_prefix) + [self.args.dokuwiki_root.absolute().name, now.strftime('%Y-%m-%d_%H-%M-%SZ') + '.tar.xz'])
    output_path_temp = output_dir / (output_filename + '.tmp')

    output_dir.mkdir(parents=True, exist_ok=True)

    with tarfile.open(output_path_temp, mode='w:xz', preset=9) as tf:
      os.chdir(self.args.dokuwiki_root.parent)
      for path, dirs, files in os.walk(self.args.dokuwiki_root.name):
        relpath = os.path.relpath(path, self.args.dokuwiki_root)
        if relpath in ('data/cache', 'data/index', 'data/locks'):
          # exclude contents of cache directories
          dirs.clear()
          files.clear()

        for name in dirs:
          tf.add(os.path.join(path, name), recursive=False)

        for name in files:
          if self.args.strip_ips and relpath.startswith('data/') and os.path.splitext(name)[1] in ('.changes', '.meta'):
            file_path = os.path.join(path, name)
            info = tf.gettarinfo(file_path)
            if name.endswith('.changes'):
              filtered_file, info.size = filter_changes_file(file_path)
            elif name.endswith('.meta'):
              filtered_file, info.size = filter_meta_file(file_path)
            tf.addfile(info, filtered_file)
          elif relpath == 'conf' and name == 'users.auth.php' and self.args.exclude_users:
            info = tf.gettarinfo(os.path.join(path, name))
            empty_users_auth_file, info.size = make_empty_users_auth_file()
            tf.addfile(info, empty_users_auth_file)
          else:
            tf.add(os.path.join(path, name), recursive=False)

    output_path_temp.rename(output_path_temp.with_name(output_filename))

if __name__ == '__main__':
  DokuwikiBackuper().run()


from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score
import numpy as np
from sklearn.model_selection import cross_val_score
from sklearn import preprocessing

class KnnClassifier:
    def do_knn_train(self, data_set_train, class_col, k):
        X_train, Y_train = self.__split(data_set_train,
                                        class_col)
        normalized_X = preprocessing.normalize(X_train)

        cv_score = []
        knn_classifiers = []
        for k_value in k:
            classifier = KNeighborsClassifier(n_neighbors=k_value)
            classifier.fit(X_train, Y_train)
            scores = cross_val_score(classifier, normalized_X, Y_train, cv=10, scoring="accuracy")
            cv_score.append(np.mean(scores))
            knn_classifiers.append(classifier)
        return dict(zip(k, knn_classifiers)), dict(zip(k, cv_score))

    def do_knn_train_single(self, data_set_train, class_col, k):
        X_train, Y_train = self.__split(data_set_train,
                                        class_col)
        normalized_X = preprocessing.normalize(X_train)
        classifier = KNeighborsClassifier(n_neighbors=k)
        classifier.fit(normalized_X, Y_train)
        return classifier

    @staticmethod
    def do_knn_test(data_set_test, classifier):
        y_pred = classifier.predict(data_set_test)
        return y_pred

    @staticmethod
    def do_knn_test_single(data_set_test, classifier):
        y_pred = classifier.predict(data_set_test)
        probs = classifier.predict_proba(data_set_test)
        return y_pred, probs[:, 1]

    @staticmethod
    def calculate_decision_metric(y_true, y_pred):
        confusion_mat = confusion_matrix(y_true, y_pred)
        # tn, fp, fn, tp = confusion_mat.ravel()
        pred_accu = accuracy_score(y_true, y_pred)
        return confusion_mat, pred_accu

    @staticmethod
    def __split(arr, class_col):
        return arr[:, 0:class_col], arr[:, class_col]

    @staticmethod
    def read_data_set(file_name):
        return np.loadtxt(file_name)


# 임의로 만들어준 html파일에서 크롤링하기

from bs4 import BeautifulSoup

# with open('./resources/exam1.html') as fp:
# C:\Users\admin\Desktop\python\resources
# with open('C:\\Users\\admin\\Desktop\\python_crawling\\python_crawling\\resources\\exam1.html') as fp:
with open('C:\\Users\\admin\\Desktop\\python\\templates\\exam1.html') as fp:
    soup = BeautifulSoup(fp, 'html.parser') # fp 는 파일을 가져오기

    # div태그 첫번째 찾기
    first_divs = soup.find("div")
    # print(first_divs)

    # div태그 전체 찾기
    all_divs = soup.find_all("div")
    soup.find_all("div", {"class":"first"})
    # <div class="first"> => div의 클래스중에 first 클래스를 찾아라

    # print(all_divs)
    for tmp in all_divs:
        print(tmp)
        all_p = tmp.find_all("p")
        for tmp1 in all_p:
            print(tmp1)
            print(tmp1.text)


import json

import requests
from requests.auth import HTTPBasicAuth
from aiogram import types

from loader import bot
from data.config import env

TEST_REQUEST_URL = env.str('TEST_PAYMENT_URL')
PAYMENT_URL = env.str('PAYMENT_URL')

def get_payment_link(amount_rub: int, description: str, user_id: int, invoice_id: int, json_data: dict):
    data = {
        "Amount": amount_rub,
        "Currency": "RUB",
        "Description": description,
        "AccountId": str(user_id),
        "InvoiceId": invoice_id,
        "JsonData": json.dumps(json_data),
    }
    result = requests.post(
        PAYMENT_URL,
       params=data,
       auth=HTTPBasicAuth(env.str("PAYMENT_PUBLIC_ID"), env.str("PAYMENT_API_SECRET"))
    )
    result = result.json()
    if result.get("Success"):
        return result.get("Model").get("Url")
    return False

def get_invoice_data(chat_id: int, title: str, description: str, payload: str,
                     amount_rub: int) -> dict:
    return {
        "chat_id": chat_id,
        "title": title,
        "description": description,
        "payload": payload,
        "provider_token": env.str("YOOKASSA_TOKEN"),
        "currency": "RUB",
        "start_parameter": "test",
        "prices": [types.LabeledPrice(label="руб", amount=amount_rub)]
    }

async def send_invoice(chat_id: int, title: str, description: str, payload: str,
                       amount_rub: int):
    await bot.send_invoice(
        chat_id=chat_id,
        title=title,
        description=description,
        payload=payload,
        provider_token=env.str("YOOKASSA_TOKEN"),
        start_parameter="true",
        currency="RUB",
        prices=[types.LabeledPrice(label="руб", amount=amount_rub * 100)]
    )


import sys
import collections
input = sys.stdin.readline

if __name__ == "__main__":
    answer = 0
    R, C = map(int, input().split(" "))
    word_list = collections.deque(list(collections.deque([]) for i in range(C)))
    first_word = input().rstrip()
    for i in range(1, R) :
        tmep = input().rstrip()
        for j in range(C) :
            word_list[j].append(tmep[j])

    for i in range(C -1) :
        w_sets = set()
        j = 0
        for words in word_list :
            w_sets.add("".join(words))
            word_list[j].popleft()
            j += 1
        if len(w_sets) == C :
            answer += 1
        else :
            break
    print(answer)


from __future__ import print_function

from typing import List, Any, Dict, Tuple, Optional, TYPE_CHECKING

if TYPE_CHECKING:
    pass

import datetime
import logging
import random
import threading

class TaskQuantum(object):
    def __init__(self, date_started, date_stopped, override_date_started=None, exit_code=None):
        self.date_started = date_started  # type: datetime.datetime
        self.date_stopped = date_stopped  # type: datetime.datetime
        self.override_date_started = override_date_started  # type: datetime.datetime
        self.exit_code = exit_code  # type: int

    @property
    def virtual_started(self):
        # type:() -> datetime.datetime
        return self.override_date_started or self.date_started

    @virtual_started.setter
    def virtual_started(self, value):
        # type: (datetime.datetime) -> None
        self.override_date_started = value

    @property
    def virtual_stopped(self):
        # VV: A task-quantum might still be in flight, assume that it will take
        #     less than 10 years to complete
        return self.date_stopped or (self.date_started + datetime.timedelta(days=365 * 10))

class ComponentDescriptor(object):
    def __init__(self, experiment, stage, component,
                 backend,
                 executable,
                 repeat_interval,
                 ranksPerNode, numberProcesses, numberThreads,
                 walltime, date_initialized, is_active):
        self.experiment = experiment  # type: str
        self.stage = stage  # type: str
        self.component = component  # type: str
        self.backend = backend  # type: str
        self.repeat_interval = repeat_interval  # type: bool
        self.ranksPerNode = ranksPerNode  # type: int
        self.numberProcesses = numberProcesses  # type: int
        self.numberThreads = numberThreads  # type: int
        self.walltime = walltime  # type: float
        self.executable = executable  # type: str
        self.date_initialized = date_initialized  # type: datetime.datetime
        self.is_active = is_active

    @property
    def is_repeat(self):
        return self.repeat_interval is not None

    @property
    def reference(self):
        return '%s.%s' % (self.stage, self.component)

    def to_dict(self):
        date_initialized = self.date_initialized.strftime("%d%m%y-%H%M%S.%f") if self.date_initialized else None
        return {
            'experiment': self.experiment,
            'stage': self.stage,
            'component': self.component,
            'backend': self.backend,
            'repeat-interval': self.repeat_interval,
            'ranksPerNode': self.ranksPerNode,
            'numberProcesses': self.numberProcesses,
            'numberThreads': self.numberThreads,
            'walltime': self.walltime,
            'executable': self.executable,
            'date-initialized': date_initialized
        }

    @staticmethod
    def from_dict(dictionary):
        experiment = dictionary['experiment']
        stage = dictionary['stage']
        component = dictionary['component']
        backend = dictionary['backend']
        repeat_interval = dictionary['repeat-interval']
        ranksPerNode = dictionary['ranksPerNode']
        numberProcesses = dictionary['numberProcesses']
        numberThreads = dictionary['numberThreads']
        walltime = dictionary['walltime']
        executable = dictionary['executable']
        date_initialized = dictionary['date-initialized']

        if date_initialized is not None:
            date_initialized = datetime.datetime.strptime(date_initialized, "%d%m%y-%H%M%S.%f")

        return ComponentDescriptor(experiment=experiment,
                                   stage=stage,
                                   component=component,
                                   backend=backend,
                                   executable=executable,
                                   repeat_interval=repeat_interval,
                                   ranksPerNode=ranksPerNode,
                                   numberProcesses=numberProcesses,
                                   numberThreads=numberThreads,
                                   walltime=walltime,
                                   date_initialized=date_initialized,
                                   is_active=False)

class Measurement(object):
    def __init__(self, component, date_last_exec, date_last_success, dur_last_exec,
                 time_execute=None,
                 time_queue=None,
                 exit_code=None,
                 date_queued=None,
                 cost_queue_pending=None,
                 cost_queue_running=None,
                 cost_queue_finished=None,
                 prod_time_avg=None,
                 prod_time_succeed_sum=None,
                 prod_time_succeed_max=None,
                 prod_percent_running=None,
                 times_failed_since_last_success=0,
                 exec_time_since_last_success=None):
        """Contains information for a single Run of some Component.

        This information is not all available at the same time. See parameter description
        for more information.

        RepeatingComponent life cycle:
        [I] <Wait till there's input> -> [QS] Queue -> [RS] Run [RD]-> Wait(repeat interval)
        -> Repeat

        * [I] -> Event instantiated (only emitted once)
        * [QS] -> Queueing started (emitted multiple times)
        * [RS] -> Running started (emitted multiple times, but possibly NOT when the
                actual event took place. We know for sure it will be emitted before
                RD (running done))
        * [RD] -> Emitted some time after the Run finishes (there might be a significant delay)

        Notice that events might be emitted with significant delays, so this class
        should NOT directly poll datetime to produce timestamps it should rely on
        method parameters.
        """
        self.time_execute = time_execute  # type: float
        self.time_queue = time_queue  # type: float
        self.component = component  # type: ComponentDescriptor
        self.date_last_exec = date_last_exec  # type: datetime.datetime
        self.date_last_success = date_last_success  # type: datetime.datetime
        self.exit_code = exit_code  # type: int
        self.dur_last_exec = dur_last_exec  # type: float

        self.date_queued = date_queued  # type: datetime.datetime

        self.cost_queue_finished = cost_queue_finished  # type: float
        self.cost_queue_running = cost_queue_running  # type: float
        self.cost_queue_pending = cost_queue_pending  # type: float

        self.original_cost_pending = cost_queue_pending  # type: float
        self.original_cost_running = cost_queue_running  # type: float
        self.original_cost_finished = cost_queue_finished  # type: float

        self.prod_time_avg = prod_time_avg  # type: float
        self.prod_time_succeed_sum = prod_time_succeed_sum # type: float
        self.prod_percent_running = prod_percent_running  # type: float
        self.prod_time_succeed_max = prod_time_succeed_max  # type: float

        self.times_failed_since_last_success = times_failed_since_last_success  # type: int
        self.exec_time_since_last_success = exec_time_since_last_success

        # VV: This is filled in IF the exit-code is not None once we find out the effective-repeat-interval
        #     ( which takes place in modelist.event_run_stop())
        self.task_quantum = None  # type: Optional[TaskQuantum]

    def dt_last_exec(self, to_date):
        # type: (datetime.datetime) -> float
        if to_date is not None:
            if self.date_last_exec is not None:
                return (to_date - self.date_last_exec).total_seconds()
            # VV: Assume that the last successful run finished just before this Component
            #     got initialized
            if self.component.date_initialized is not None:
                return (to_date - self.component.date_initialized).total_seconds()

    def dt_last_success(self, to_date):
        # type: (datetime.datetime) -> float
        if to_date is not None:
            if self.date_last_success is not None:
                return (to_date - self.date_last_success).total_seconds()

            # VV: Assume that the last run finished just before this Component
            #     got initialized
            if self.component.date_initialized is not None:
                return (to_date - self.component.date_initialized).total_seconds()

    @property
    def resource_cost_estimated(self):
        return (self.component.numberProcesses
                * self.component.numberThreads
                * self.component.walltime)

    @property
    def resource_cost_measured(self):
        return (self.component.numberProcesses
                * self.component.numberThreads
                * self.time_execute)

    @property
    def date_started(self):
        # type: () -> datetime.datetime
        if self.date_queued is not None and self.time_queue is not None:
            return self.date_queued + datetime.timedelta(seconds=self.time_queue)

    @property
    def date_stopped(self):
        # type: () -> datetime.datetime
        date_started_run = self.date_started
        if self.time_execute is not None and date_started_run is not None:
            return date_started_run + datetime.timedelta(seconds=self.time_execute)

class Modelist(object):
    def __init__(self):
        self.log = logging.getLogger('Modelist')

        self.metadata = {
        }  # type: Dict[ComponentDescriptor, EngineMetaData]

        self.producers_of = {

        }  # type: Dict[ComponentDescriptor, List[ComponentDescriptor]]

        self.consumers_of = {

        }  # type: Dict[ComponentDescriptor, List[ComponentDescriptor]]

        self.descriptors = {
            # VV: experiment_name : {
            #       stage_name : {
            #         component_name : ComponentDescriptor
            #     }
            # }
        }  # type: Dict[str, Dict[str, Dict[str, ComponentDescriptor]]]

        self.recorded_measurements = {
            # VV: component: { List[Measurement (chronological order)] }
        }  # type: Dict[ComponentDescriptor, List[Measurement]]

        self.backend_state = {
            # VV: FIXME add information about WHICH backend is used:
            'queued': {},  # type: Dict[ComponentDescriptor, datetime.datetime]
            'running': {},  # type: Dict[ComponentDescriptor, datetime.datetime]
            'finished': []  # type: List[Tuple[datetime.datetime, ComponentDescriptor, float]]
        }

    def desc_from_reference(self, reference, experiment_name):
        stage, name = reference.split('.')
        try:
            desc = self.descriptors[experiment_name][stage][name]
            return desc
        except KeyError:
            return None

    def register_component(self, experiment, stage, component,
                           backend,
                           executable,
                           repeat_interval,
                           ranksPerNode, numberProcesses, numberThreads,
                           walltime,
                           date_initialized,
                           producer_descriptors=None):
        desc = ComponentDescriptor(experiment, stage, component,
                                            backend,
                                            executable,
                                            repeat_interval,
                                            ranksPerNode, numberProcesses, numberThreads,
                                            walltime, date_initialized, True)

        if experiment not in self.descriptors:
            self.descriptors[experiment] = {}

        if stage not in self.descriptors[experiment]:
            self.descriptors[experiment][stage] = {}

        assert component not in self.descriptors[experiment][stage]

        assert desc not in self.producers_of

        self.producers_of[desc] = producer_descriptors

        if desc not in self.consumers_of:
            self.consumers_of[desc] = []

        for producer in producer_descriptors:
            if producer not in self.consumers_of:
                self.consumers_of[producer] = [desc]
            else:
                self.consumers_of[producer].append(desc)

        # VV: All we know about this desc is that this is the first time we come across it, and
        #     its repeat_interval (if any)
        meta = EngineMetaData(
            None,
            None,
            None,
            None,
            repeat_interval,
            0
        )
        assert desc not in self.metadata

        self.metadata[desc] = meta

        self.descriptors[experiment][stage][component] = desc

        self.recorded_measurements[desc] = []

        return desc

    def event_queue_start(self, experiment, stage, component, date_now):
        #  type: (str, str, str, datetime.datetime) -> None
        self.log.critical('event_queue_start %s %s %s' % (experiment, stage, component))
        desc = self.descriptors[experiment][stage][component]
        safe_to_increase_cost = True

        assert desc not in self.backend_state['queued']

        self.backend_state['queued'][desc] = date_now

        if self.recorded_measurements[desc]:
            # VV: find the last measurement that actually ran

            for last_meas in self.recorded_measurements[desc][::-1]:
                if last_meas.exit_code is not None:
                    break
            else:
                # VV: Should we ever reach this point, we just have to use the very first
                #     measurement even though it crashed because we can use ITS last success/last run
                #     (recall that we faked these values under register_component)
                last_meas = self.recorded_measurements[desc][0]

            last_success = None
            last_exec = None
            dur_last_exec = None
            exec_time_since_last_success = 0

            if last_meas.exit_code == 0:
                # VV: Last run was successful
                last_exec = last_meas.date_stopped
                dur_last_exec = last_meas.time_execute
                last_success = last_meas.date_stopped
                exec_time_since_last_success = 0
            elif last_meas.exit_code is not None:
                # VV: Last run was not successful
                last_exec = last_meas.date_stopped
                dur_last_exec = last_meas.time_execute
                last_success = last_meas.date_last_success
                exec_time_since_last_success = last_meas.exec_time_since_last_success + dur_last_exec
            elif last_meas.exit_code is None:
                # VV: Component did not launch the last time we tried. Pretend that this never
                #     happened and re-use whatever info the failed-to-launch run used
                last_exec = last_meas.date_last_exec
                dur_last_exec = last_meas.dur_last_exec
                last_success = last_meas.date_last_success

            # VV: Sanitize
            dur_last_exec = max(0.0, dur_last_exec)

            meas = Measurement(
                component=desc,
                date_last_exec=last_exec,
                date_last_success=last_success,
                dur_last_exec=dur_last_exec,
                date_queued=date_now,
                exec_time_since_last_success=exec_time_since_last_success
            )
        else:
            # VV: This is the beginning of the very first measurement,
            #     fake some data about the previous, and completely imagined, run
            meas = Measurement(
                component=desc,
                date_last_exec=desc.date_initialized,
                date_last_success=desc.date_initialized,
                dur_last_exec=0,
                date_queued=date_now,
                exec_time_since_last_success=0
            )

        self.recorded_measurements[desc].append(meas)

    def event_run_start(self, experiment, stage, component, date_now):
        #  type: (str, str, str, datetime.datetime) -> None
        desc = self.descriptors[experiment][stage][component]

        if desc not in self.backend_state['queued']:
            self.log.critical("%s is queued:%s, running: %s, finished:%s" % (
                desc.reference,
                desc in self.backend_state['queued'],
                desc in self.backend_state['running'],
                desc in self.backend_state['finished'],
            ))

        assert desc in self.backend_state['queued']

        del self.backend_state['queued'][desc]

        self.backend_state['running'][desc] = date_now

        # VV: We know that the component started running at "date_now",
        #    now we can update the queue time
        current = self.recorded_measurements[desc][-1]
        current.time_queue = (date_now - current.date_queued).total_seconds()

        # VV: Mark this TaskQuantum to be 'in-flight' (we don't know when it'll finish
        #    , see quantum.virtual_stopped)
        self.metadata[desc].task_quantums.append(
            TaskQuantum(date_started=date_now, date_stopped=None)
        )

    def compute_effective_delta_time(self, desc, to_date):
        # type: (ComponentDescriptor, datetime.datetime) -> Dict[str, float]
        if self.metadata[desc].last_success_stopped:
            """ VV: Schematic to understand the code
                    (i.e earliest_date_lookup and task_quantum_to_exec_time)
            n = not accounted by this run
            y = accounted by this run
            c = time-step within this run (could be failed or successful)
            l = time-step within last successful run
            f = past FAILED run (there's a chance it doesn't exist because previous run was successful
                                 doesn't change a thing in the algorithm; it's only shown for brevity)

            VV: So what about producer runs which started WHILE the consumer was running ?
                Well we can't know about these before attempting to execute the consumer, in fact
                we may event want to dismiss information which we found out after we Submitted the
                Task to the backend because we didn't know about it when we made the decision to
                execute the Task at hand.
            VV: FIXME Look at the above comment --^

            -nnnnnyyyyyyy---yyyynnnnnn----nnnnnnn--nn-  <- producer quantums (even future ones)
            ------llll---fffff--cccccccccccccc--------  <- desc
                     ^          ^            ^
                     |  when_run_started     |
                     |                   date_now
            last_success_stopped
            """

            def earliest_date_lookup(task_quantum, to_date):
                # type: (TaskQuantum, datetime.datetime) -> bool
                assert task_quantum.date_stopped is None or (
                        task_quantum.virtual_started <= task_quantum.date_stopped
                )

                return (task_quantum.date_started < to_date) and (
                        task_quantum.date_stopped is None
                        or self.metadata[desc].last_success_started < task_quantum.date_stopped
                )
        else:
            def earliest_date_lookup(task_quantum, to_date):
                # type: (TaskQuantum, datetime.datetime) -> bool
                return task_quantum.date_started < to_date

        def quantum_to_observed_exec_time(task_quantum, to_date):
            # type: (TaskQuantum, datetime.datetime) -> Optional[Tuple[float, bool]]
            success = task_quantum.exit_code == 0

            if earliest_date_lookup(task_quantum, to_date):
                if task_quantum.date_stopped is None or task_quantum.date_stopped > to_date:
                    cutoff_high = to_date
                else:
                    cutoff_high = task_quantum.date_stopped

                if self.metadata[desc].last_success_started:
                    # VV: If we've run before then make sure to ONLY account for the time since the
                    #     beginning of this batch of tasks (which is after the last success started)
                    if self.metadata[desc].last_success_started > task_quantum.date_started:
                        cutoff_low = self.metadata[desc].last_success_started
                    else:
                        cutoff_low = task_quantum.date_started
                else:
                    # VV: This engine has never run before so there's no lower bound in the task-quantum
                    #     starting date
                    cutoff_low = task_quantum.date_started

                return (cutoff_high-cutoff_low).total_seconds(), success
            else:
                return None

        prod_time_min = 0
        prod_time_sum = 0
        prod_time_avg = 0
        prod_time_max = 0
        prod_time_unknown_sum = 0
        prod_time_succeed_sum = 0
        prod_time_succeed_max = 0
        prod_percent_running = 0
        prod_percent_done = 0

        def is_producer_relevant(producer):
            # type: (ComponentDescriptor) -> bool

            origin = self.metadata[desc].last_success_stopped

            if origin is None or self.metadata[producer].is_active(origin):
                return True

            return False

        relevant_producers = list(filter(is_producer_relevant, self.producers_of[desc]))

        for k in self.producers_of[desc]:
            if k not in relevant_producers:
                self.log.debug("Accumulated time (prod %s is irrelevant to %s)" % (
                    k.reference, desc.reference
                ))

        time_fail = {}  # type: Dict[ComponentDescriptor, float]
        time_succeed = {}  # type: Dict[ComponentDescriptor, float]
        time_total = {}  # type: Dict[ComponentDescriptor, float]
        time_unknown = {}  # type: Dict[ComponentDescriptor, Optional[float]]
        prod_running = {}  # type: Dict[ComponentDescriptor, bool]

        if relevant_producers:
            for prod in relevant_producers:
                time_fail[prod] = 0.0
                time_succeed[prod] = 0.0
                time_total[prod] = 0.0
                # VV: Leave this to None so that the assertion bellow checks for the correctness of the
                #     algorithm
                time_unknown[prod] = None
                prod_running[prod] = False
                for task_quantum in list(self.metadata[prod].task_quantums):
                    ret_tuple = quantum_to_observed_exec_time(task_quantum, to_date)
                    if ret_tuple is not None:
                        ret, success = ret_tuple

                        # VV: Backends might extremely small values
                        ret = max(0.5, ret)

                        # VV: Source Engines will always be treated as 'unknown' unless we force
                        #     them to register as Successes
                        if task_quantum.date_stopped is not None \
                                or prod.is_repeat is False:
                            # VV: Need to be able to inspect the task finish (date_stopped) BEFORE `to_date`
                            if success or prod.is_repeat is False:
                                time_succeed[prod] += ret
                            else:
                                time_fail[prod] += ret
                        else:
                            # VV: There can be ONLY one task in transit per producer
                            if time_unknown[prod] is not None:
                                raise ValueError("Producer %s of %s has multiple in-transit tasks" % (
                                    prod.reference, desc.reference
                                ))

                            time_unknown[prod] = ret

                        if task_quantum.date_stopped is None or task_quantum.date_stopped > to_date:
                            prod_running[prod] = True

                time_unknown[prod] = time_unknown[prod] or 0.0
                time_total[prod] = time_fail[prod] + time_succeed[prod] + time_unknown[prod]

            prod_percent_done = (
                    (len(self.producers_of[desc]) - len(relevant_producers)) / float(len(self.producers_of[desc]))
            )

            prod_percent_running = sum(prod_running.values()) / float(len(relevant_producers))
            prod_time_unknown_sum = sum(time_unknown.values())
            prod_time_sum = sum(time_total.values())
            prod_time_min = min(time_total.values())
            prod_time_max = max(time_total.values())
            prod_time_avg = prod_time_sum / float(len(relevant_producers))
            prod_time_succeed_max = max(time_succeed.values())
            prod_time_succeed_sum = sum(time_succeed.values())

        return {
            'prod-time-unknown-sum': prod_time_unknown_sum,
            'prod-time-sum': prod_time_sum,
            'prod-time-min': prod_time_min,
            'prod-time-max': prod_time_max,
            'prod-time-avg': prod_time_avg,
            'prod-time-succeed-sum': prod_time_succeed_sum,
            'prod-time-succeed-max': prod_time_succeed_max,
            'prod-percent-running': prod_percent_running,
            'prod-percent-done': prod_percent_done,
            'time-total': time_total,
            'time-succeed': time_succeed,
            'time-unknown': time_unknown,
            'prod-running': prod_running,
            'time-fail': time_fail
        }

    def _copy_delta_info_to_measurement(self, measurement, delta):
        # type: (Measurement, Dict[str, float]) -> None
        measurement.prod_percent_running = delta['prod-percent-running']
        measurement.prod_time_avg = delta['prod-time-avg']
        measurement.prod_time_succeed_sum = delta['prod-time-succeed-sum']
        measurement.prod_time_succeed_max = delta['prod-time-succeed-max']

    def event_run_stop(self, experiment, stage, component, date_now, exit_code):
        # type: (str, str, str, datetime.datetime, int) -> None
        desc = self.descriptors[experiment][stage][component]

        if desc not in self.backend_state['running']:
            self.log.critical("%s is queued:%s, running: %s, finished:%s" % (
                desc.reference,
                desc in self.backend_state['queued'],
                desc in self.backend_state['running'],
                desc in self.backend_state['finished'],
            ))

        assert desc in self.backend_state['running']

        date_start = self.backend_state['running'][desc]
        actual_walltime = (date_now - date_start).total_seconds()
        self.backend_state['running'][desc] = date_now
        del self.backend_state['running'][desc]

        entry = [date_now, desc, actual_walltime]
        self.backend_state['finished'].append(entry)

        current = self.recorded_measurements[desc][-1]
        time_execute = (date_now - current.date_started).total_seconds()
        current.exit_code = exit_code
        current.time_execute = time_execute
        current.exec_time_since_last_success += time_execute

        assert self.metadata[desc].task_quantums[-1].date_stopped is None
        self.metadata[desc].task_quantums[-1].date_stopped = date_now
        self.metadata[desc].task_quantums[-1].exit_code = exit_code

        if exit_code is None or time_execute is None:
            self.metadata[desc].run_stop(date_now, exit_code)
            return

        delta_info = self.compute_effective_delta_time(desc, self.metadata[desc].when_run_started)

        self.log.info("Exit-code:%s Sum:%.2f Min:%.2f Max:%.2f Avg:%.2f "
                      "- %%Fly:%.2f %%Done:%.2f for %s" % (
                          exit_code,
                          delta_info['prod-time-sum'], delta_info['prod-time-min'], delta_info['prod-time-max'],
                          delta_info['prod-time-avg'], delta_info['prod-percent-running'],
                          delta_info['prod-percent-done'],
                          desc.reference
                      ))

        self._copy_delta_info_to_measurement(current, delta_info)

        if desc.is_repeat and exit_code == 0:
            # VV: We've just had a successful run, we can now measure the 'effective-repeat-interval'

            # VV: TODO Decide a better measurement to use as the effective-repeat-interval ?
            effective_repeat_interval = delta_info['prod-time-succeed-max']
            self.log.info("Effective interval: %s for %s" % (
                effective_repeat_interval, desc.reference
            ))

            self.metadata[desc].observed_effective_repeat_intervals.append(effective_repeat_interval)

        self.metadata[desc].run_stop(date_now, exit_code)

class EngineMetaData(object):
    def __init__(self,
                 when_queue_started,
                 when_queue_stopped,
                 when_run_started,
                 when_run_stopped,
                 delay,
                 times_run=0,
                 times_successful_run=0,
                 went_inactive_at=None):
        self.task_quantums = []  # type: List[TaskQuantum]
        self.when_queue_started = when_queue_started  # type: datetime.datetime
        self.when_queue_stopped = when_queue_stopped  # type: datetime.datetime
        self.when_run_started = when_run_started  # type: datetime.datetime
        self.when_run_stopped = when_run_stopped  # type: datetime.datetime

        self.original_delay = delay
        self.times_run = times_run  # type: int

        self.times_failed_since_success = 0
        self.times_successful_run = times_successful_run
        self.last_run_stopped = when_run_stopped
        self.last_success_started = None  # type: datetime.datetime
        self.last_success_stopped = None  # type: datetime.datetime
        self.is_repeat = (delay is not None)
        self.next_delay = None
        self.next_queue_date = None
        self.went_inactive_at = went_inactive_at

        self.observed_effective_repeat_intervals = []  # type: List[float]

    def is_active(self, when):
        # type: (datetime.datetime) -> bool
        if self.went_inactive_at is None:
            return True

        return self.went_inactive_at >= when

    def queue_start(self, when):
        self.when_queue_started = when
        self.when_queue_stopped = None
        self.when_run_started = None
        self.when_run_stopped = None
        assert when is not None

    def queue_stop(self, when):
        # type: (datetime.datetime) -> None
        assert self.when_queue_started is not None
        assert self.when_queue_stopped is None
        assert self.when_run_started is None
        assert self.when_run_stopped is None
        assert when is not None

        self.when_queue_stopped = when

    def run_start(self, when):
        # type: (datetime.datetime) -> None
        assert self.when_queue_started is not None
        assert self.when_queue_stopped is not None
        assert self.when_run_started is None
        assert self.when_run_stopped is None
        assert when is not None

        self.when_run_started = when

    def run_stop(self, when, exit_code):
        # type: (datetime.datetime, Optional[int]) -> None
        assert self.when_queue_started is not None
        assert self.when_queue_stopped is not None
        assert self.when_run_started is not None
        assert self.when_run_stopped is None
        assert when is not None

        self.when_run_stopped = when
        self.last_run_stopped = when
        self.times_run += 1

        if exit_code == 0:
            self.times_successful_run += 1

            self.last_success_started = self.when_run_started
            self.last_success_stopped = self.when_run_stopped
            self.times_failed_since_success = 0
        else:
            self.times_failed_since_success += 1

    def register_next_delay(self, next_delay):
        assert self.is_repeat

        self.next_delay = next_delay

    def register_next_queue_date(self, next_queue_date):
        self.next_queue_date = next_queue_date

class OptimiserDecisionMetadata(object):
    def __init__(self, desc, exploit_chance, exploit_target, explore_target_low, explore_target_high):
        # type: (ComponentDescriptor, float, float, float, float) -> None
        self._last_recorded_times_run = None  # type: Optional[int]
        self._exploit_chance = exploit_chance

        # VV: True = Exploit, False = Explore
        self._should_exploit_next = False
        self._desc = desc

        self._exploit_target_factor = exploit_target
        self._explore_target_low = explore_target_low
        self._explore_target_high = explore_target_high
        self._date_last_stopped = None
        self._explore_target_factor = None

        self._date_to_submit = None  # type: Optional[datetime.datetime]

        self._is_dirty = True

        self.target_effective_repeat_interval = None  # type: Optional[float]
        self.attempts = 0

    def should_exploit(self, metadata):
        # type: (EngineMetaData) -> bool

        if metadata.times_run != self._last_recorded_times_run:
            self._should_exploit_next = (random.uniform(0.0, 1.0) <= self._exploit_chance)

            if self._should_exploit_next is False:
                self._explore_target_factor = random.uniform(
                    self._explore_target_low, self._explore_target_high
                )

        self._last_recorded_times_run = metadata.times_run

        return self._should_exploit_next

    def explore_target_factor(self):
        return self._explore_target_factor

    def exploit_target_factor(self):
        return self._exploit_target_factor

    def register_choice(self, date_to_submit, date_last_stopped):
        # type: (datetime.datetime, datetime.datetime) -> None
        self._date_to_submit = date_to_submit
        self._date_last_stopped = date_last_stopped
        self.is_dirty = False

    def next_repeat_interval(self):
        if self.is_dirty:
            return None

        return (self._date_to_submit - self._date_last_stopped).total_seconds()

    @property
    def is_dirty(self):
        # type: () -> bool
        return self._is_dirty

    @is_dirty.setter
    def is_dirty(self, value):
        # type: (bool) -> None
        self._is_dirty = value

        if value is True:
            self._date_to_submit = None

class Optimizer(object):
    def __init__(self, experiment_name, modelist=None):
        if modelist is None:
            modelist = Modelist()

        self.modelist = modelist

        # VV: All active ComponentDescriptors and their Consumer references
        #     ordered based on when they were submitted for scheduling (lower index = earlier)
        self.active_ordered = []  # type: List[ComponentDescriptor]

        self.lock_engine_meta_data = threading.RLock()

        # VV: @tag: OPTIMIZER-observe-engine-updates
        # VV: There's a small chance an engine emits a task-event twice!
        self.past_event_hashes = set()  # type: set[Tuple[int, str, str, datetime.datetime]]
        self.engine_metadata_pending = [
            # VV: stage, component_name,
            #     initialized|queue_start|queue_stop|run_start|run_stop|finished_completely,
            #     when,
            #     exit-code|None|[backend, executable, repeat_interval, ... (all from register_component)...]
        ]  # type: List[Tuple[ComponentDescriptor, str, datetime.datetime, Any]]

        self.engine_metadata = self.modelist.metadata
        self.producers_of = self.modelist.producers_of
        self.consumers_of = self.modelist.consumers_of

        self.experiment_name = experiment_name

        self.lock = threading.RLock()

        self.log = logging.getLogger('Optimizer')

        self.decisions = {

        }  # type: Dict[ComponentDescriptor, OptimiserDecisionMetadata]

    def desc_from_reference(self, reference):
        return self.modelist.desc_from_reference(reference, self.experiment_name)

    def register_component(self, stage, component,
                           backend,
                           executable,
                           repeat_interval,
                           ranksPerNode, numberProcesses, numberThreads,
                           walltime,
                           date_initialized,
                           producer_references,
                           exploit_chance,
                           exploit_target,
                           explore_target_low,
                           explore_target_high
                           ):

        exploit_chance = exploit_chance if exploit_chance is not None else 0.9
        exploit_target = exploit_target if exploit_target is not None else 0.75
        explore_target_low = explore_target_low if explore_target_low is not None else 0.25
        explore_target_high = explore_target_high if explore_target_high is not None else 0.5

        entry = [
            stage, component,
            'initialized',
            date_initialized,
            (backend,
             executable,
             repeat_interval,
             ranksPerNode, numberProcesses, numberThreads,
             walltime,
             date_initialized,
             producer_references,
             exploit_chance,
             exploit_target,
             explore_target_low,
             explore_target_high
             )
        ]

        with self.lock_engine_meta_data:
            self.engine_metadata_pending.append(entry)

    def _register_component(self, stage, component,
                            backend,
                            executable,
                            repeat_interval,
                            ranksPerNode, numberProcesses, numberThreads,
                            walltime,
                            date_initialized,
                            producer_references,
                            exploit_chance,
                            exploit_target,
                            explore_target_low,
                            explore_target_high):

        def stage_name_to_index(my_stage):
            # type: (str) -> int
            prefix = 'stage'
            assert my_stage.startswith(prefix)
            index = int(my_stage[len(prefix):])
            del prefix

            return index

        # VV: TODO Does it make sense to keep track of Source components too ?
        my_stage = stage_name_to_index(stage)

        in_stage_deps = [ref for ref in producer_references if my_stage <= stage_name_to_index(ref.split('.')[0])]

        producer_descs = []
        for dep in in_stage_deps:
            descr = self.desc_from_reference(dep)
            if descr is None:
                raise ValueError(
                    "InStageDependency %s of %s.%s cannot be mapped to an existing Producer description" % (
                        dep, stage, component,
                ))
            producer_descs.append(descr)

        desc = self.modelist.register_component(self.experiment_name, stage, component,
                                                backend,
                                                executable,
                                                repeat_interval,
                                                ranksPerNode, numberProcesses, numberThreads,
                                                walltime,
                                                date_initialized,
                                                producer_descriptors=producer_descs)

        assert desc not in self.active_ordered
        self.active_ordered.append(desc)

        self.decisions[desc] = OptimiserDecisionMetadata(desc,
                                                         exploit_chance,
                                                         exploit_target,
                                                         explore_target_low, explore_target_high)

    def event_queue_start(self, stage, component, when):
        entry = [
            stage, component,
            'queue_start',
            when,
            None
        ]
        event_hash = (stage, component, 'queue_start', when)

        with self.lock_engine_meta_data:
            if event_hash not in self.past_event_hashes:
                self.past_event_hashes.add(event_hash)
                self.engine_metadata_pending.append(entry)
            else:
                self.log.warning("Received a duplicate event %s" % entry)

    def _event_queue_start(self, stage, component, when):
        with self.lock:
            self.modelist.event_queue_start(self.experiment_name, stage, component, when)
            desc = self.modelist.descriptors[self.experiment_name][stage][component]
            self.engine_metadata[desc].queue_start(when)

    def event_queue_stop(self, stage, component, when):
        entry = [
            stage, component,
            'queue_stop',
            when,
            None
        ]

        event_hash = (stage, component, 'queue_stop', when)

        with self.lock_engine_meta_data:
            if event_hash not in self.past_event_hashes:
                self.past_event_hashes.add(event_hash)
                self.engine_metadata_pending.append(entry)
            else:
                self.log.warning("Received a duplicate event %s" % entry)

    def _event_queue_stop(self, stage, component, when):
        with self.lock:
            desc = self.modelist.descriptors[self.experiment_name][stage][component]
            self.engine_metadata[desc].queue_stop(when)

    def event_run_start(self, stage, component, when):
        entry = [
            stage, component,
            'run_start',
            when,
            None
        ]

        event_hash = (stage, component, 'run_start', when)

        with self.lock_engine_meta_data:
            if event_hash not in self.past_event_hashes:
                self.past_event_hashes.add(event_hash)
                self.engine_metadata_pending.append(entry)
            else:
                self.log.warning("Received a duplicate event %s" % entry)

    def _event_run_start(self, stage, component, when):
        with self.lock:
            self.modelist.event_run_start(self.experiment_name, stage, component, when)
            desc = self.modelist.descriptors[self.experiment_name][stage][component]
            self.engine_metadata[desc].run_start(when)

    def event_run_stop(self, stage, component, when, exit_code):
        entry = [
            stage, component,
            'run_stop',
            when,
            exit_code
        ]

        event_hash = (stage, component, 'run_stop', when)

        with self.lock_engine_meta_data:
            if event_hash not in self.past_event_hashes:
                self.past_event_hashes.add(event_hash)
                self.engine_metadata_pending.append(entry)
            else:
                self.log.warning("Received a duplicate event %s" % entry)

    def _decide_next_effective_repeat_interval(self, desc, date_last_started, date_last_stopped, exit_code):
        metadata = self.engine_metadata[desc]
        decision = self.decisions[desc]

        def guess_next_effective_repeat_interval():
            max_recent = 3
            values = metadata.observed_effective_repeat_intervals[-max_recent:]

            if len(values) > 1:
                weight_max = 2
                weight_min = 1

                weights = [idx * (weight_max - weight_min) / (len(values) - 1.0) + weight_min for idx in range(len(values))]

                guess = sum([w_v[0] * w_v[1] for w_v in zip(weights, values)]) / float(sum(weights))
            elif len(values) == 1:
                guess = values[0]
            else:
                guess = None

            return guess

        if exit_code == 0:
            # VV: Modelist populates observed_effective_repeat_interval

            guess = guess_next_effective_repeat_interval()

            if guess is None:
                self.log.critical("%s had a successful task but no observed "
                                  "effective-repeat interval" % desc.reference)
                assert False

            self.log.info("%s next-eri: %.2f because it has just succeeded on %s" % (
                desc.reference, guess, date_last_stopped
            ))
            return guess
        else:
            # VV: We've just failed find out how long we waited for the producer to execute
            #     for and increase that target eri accordingly
            prod_info = self.modelist.compute_effective_delta_time(desc, date_last_started)
            current_progress = prod_info['prod-time-succeed-max']

            guess = guess_next_effective_repeat_interval()

            if guess is not None and guess < current_progress:
                self.log.info("%s keeping next-eri: %.2f because it failed on %s %d times and tested out %.2f" % (
                    desc.reference, guess, date_last_stopped, decision.attempts, current_progress
                ))
                return guess

            if current_progress <= 10 * 60:
                delta = min(60.0, current_progress * 1.5)
            elif current_progress <= 60 * 60:
                delta = min(60.0 * 10, current_progress * 1.5)
            elif current_progress <= 12 * 60 * 60:
                delta = 60.0 * 30
            elif current_progress <= 24 * 60 * 60:
                delta = 60.0 * 60
            else:
                # VV: Looks like a huge job, try every 4 hours
                delta = min(60.0 * 60 * 4, current_progress * 1.5)

            delta = max(1.0, delta)

            guess = current_progress + delta

            self.log.info("%s next-eri: %.2f because it failed on %s %d times" % (
                desc.reference, guess, date_last_stopped, decision.attempts
            ))
            return guess

    def _event_run_stop(self, stage, component, when, exit_code):
        with self.lock:
            self.modelist.event_run_stop(self.experiment_name, stage, component, when, exit_code)

            desc = self.modelist.descriptors[self.experiment_name][stage][component]

            if desc.is_repeat:
                decision = self.decisions[desc]
                if exit_code == 0:
                    self.log.info("It took %d attempts before %s succeeded" % (
                        decision.attempts, desc.reference
                    ))

                    # VV: We have just completed successfully, reset the last decision
                    decision.is_dirty = True
                    decision.target_effective_repeat_interval = None
                    decision.attempts = 0
                else:
                    decision.attempts += 1
                metadata = self.engine_metadata[desc]
                if metadata.is_active(when):
                    # VV: Given that we've just succeeded/failed pick the next repeat-interval
                    target_eri = self._decide_next_effective_repeat_interval(desc, metadata.when_run_started,
                                                                             when, exit_code)
                    assert target_eri is not None
                    decision.target_effective_repeat_interval = target_eri
                else:
                    self.log.info("%s is not a candidate for optimization" % desc.reference)

    def event_finished_completely(self, stage, component, when):
        entry = [
            stage, component,
            'finished_completely',
            when,
            None # VV: No payload
        ]

        with self.lock_engine_meta_data:
            self.engine_metadata_pending.append(entry)

    def _event_finished_completely(self, stage, component, when):
        with self.lock:
            desc = self.modelist.descriptors[self.experiment_name][stage][component]
            self.engine_metadata[desc].went_inactive_at = when
            desc.is_active = False
            self.log.info("Optimizer will now stop optimizing %s (event received at: %s)" % (
                desc.reference, when
            ))

    def optimize(self, date_now):
        with self.lock:
            # self.log.info("Digesting pending engine events")
            if self._digest_events():
                # VV: Invalidate all past decisions because we've just got new information
                for desc in self.decisions:
                    decision = self.decisions[desc]
                    metadata = self.engine_metadata[desc]
                    decision.is_dirty = (decision._last_recorded_times_run != metadata.times_run)

                self.log.info("Running the Optimizer")
                self._optimize_kernel(date_now)
                self.log.info("Optimizer done running")

    def _digest_events(self):
        # type: () -> int
        with self.lock_engine_meta_data:
            pending = self.engine_metadata_pending[:]
            self.engine_metadata_pending = []

        processed = len(pending)

        for event in pending:
            stage, component_name, event_name, when, payload = event
            self.log.info("Digesting %s for %s.%s from %s payload: %s" % (
                event_name, stage, component_name, when, payload
            ))
            if event_name == 'initialized':
                (backend,
                 executable,
                 repeat_interval,
                 ranksPerNode, numberProcesses, numberThreads,
                 walltime,
                 date_initialized,
                 producer_references,
                 exploit_chance,
                 exploit_target,
                 explore_low,
                 explore_high
                 ) = payload

                self._register_component(
                    stage, component_name,
                    backend,
                    executable,
                    repeat_interval,
                    ranksPerNode, numberProcesses, numberThreads,
                    walltime,
                    date_initialized,
                    producer_references,
                    exploit_chance,
                    exploit_target,
                    explore_low,
                    explore_high
                )
            elif event_name == 'queue_start':
                self._event_queue_start(stage, component_name, when)
            elif event_name == 'queue_stop':
                self._event_queue_stop(stage, component_name, when)
            elif event_name == 'run_start':
                self._event_run_start(stage, component_name, when)
            elif event_name == 'run_stop':
                self._event_run_stop(stage, component_name, when, payload)
            elif event_name == 'finished_completely':
                self._event_finished_completely(stage, component_name, when)
            else:
                assert False

        return processed

    def _explore_desc(self, desc, date_now, factor):
        # type: (ComponentDescriptor, datetime.datetime, float) -> Optional[Tuple[float, float]]

        # VV: Use factor to search for larger/smaller effective-repeat-intervals
        #     (small factors will steer optimiser towards looking for a smaller
        #     effective-repeat-interval)

        ret = self._target_effective_repeat_interval(desc, date_now, factor)

        if ret is not None:
            repeat_interval, eri = ret
            execute_at = self.engine_metadata[desc].last_run_stopped + datetime.timedelta(seconds=repeat_interval)

            self.log.debug("Explore ri:%.2f, eri:%.2f*%.2f for %s at %s" % (
                repeat_interval, eri, factor, desc.reference, execute_at
            ))

            return repeat_interval, eri

        return None

    def _exploit_desc(self, desc, date_now, factor):
        # type: (ComponentDescriptor, datetime.datetime, float) -> Optional[Tuple[float, float]]
        ret = self._target_effective_repeat_interval(desc, date_now, factor)

        if ret is not None:
            repeat_interval, eri = ret
            execute_at = self.engine_metadata[desc].last_run_stopped + datetime.timedelta(seconds=repeat_interval)

            self.log.debug("Exploit ri:%.2f, eri:%.2f*%.2f for %s at %s" % (
                repeat_interval, eri, factor, desc.reference, execute_at
            ))

            return repeat_interval, eri

        return None

    def _target_effective_repeat_interval(self, desc, date_now, factor, backend_delay_offset=15):
        # type: (ComponentDescriptor, datetime.datetime, float, float) -> Optional[Tuple[float, float]]
        # VV: Returns (suggested repeat interval, expected effective-repeat-interval)

        # VV: Use factor to search for larger/smaller effective-repeat-intervals
        #     (small factors will steer optimiser towards looking for a smaller
        #     effective-repeat-interval)

        metadata = self.engine_metadata[desc]  # type: EngineMetaData
        decision = self.decisions[desc]

        alive_producers = [p for p in self.producers_of[desc] if self.engine_metadata[p].is_active(date_now)]

        eff_rep_int = decision.target_effective_repeat_interval
        prod_info = self.modelist.compute_effective_delta_time(desc, date_now)

        if eff_rep_int is None:
            self.log.critical("%s does not have a target effective repeat interval" % desc.reference)

        assert eff_rep_int is not None

        if not alive_producers:
            # VV: All of the producers are done, we can't make any more choices
            metadata.register_next_delay(None)
            self.log.info("All producers of %s have finished. Will revert to default repeat-interval" % (
                desc.reference
            ))

            metadata.went_inactive_at = date_now
            desc.is_active = False
            return metadata.original_delay, eff_rep_int

        if metadata.last_run_stopped is None:
            # VV: This engine has never run before we can't optimise its repeat-interval if it's not
            #     currently attempting to repeat
            return None

        current_delay = (date_now - metadata.last_run_stopped).total_seconds()
        current_progress = prod_info['prod-time-succeed-max']

        time_succeed = prod_info['time-succeed']  # type: Dict[ComponentDescriptor, float]
        prod_running = prod_info['prod-running']  # type: Dict[ComponentDescriptor, bool]

        # VV: Some backends (e.g. LSF) introduce significant delays so we can't know for sure
        #     whether a producer is actually running right now. Set the repeat interval to
        #     expire 30 seconds after what we now believe to be adequate time to wait for the
        #     producers to execute for. This means that the Optimizer puts an artificial lower
        #     threshold to the effective-repeat interval. The advantage is that it will make
        #     fewer mistakes (as in less faild-executions). If no message arrives within the
        #     next K seconds then the optimizer will assume that everything is fine. This is
        #     best used for RepeatingEngines because plain Engines will likely not fail.
        #     K depends on a) the backend, b) the priority of the job, for the time being
        #     we'll use an optimistic delay of 15 seconds.

        if current_progress == 0 or (current_progress <= eff_rep_int * factor) and prod_running:
            # VV: Our model suggests that we should not execute right now but some of our
            #     producers are running right now, let's pick a time in the future where
            #     our prod-time-succeed-max will be good-enough

            # VV: Find the producer with the highest prod-succeed-time which is currently running
            pair_succeed = [(p, time_succeed[p]) for p in list(prod_running.keys())]
            pair_succeed_sorted = sorted(pair_succeed, key=lambda p: p[1], reverse=True)
            top_prod, top_succeed_time = pair_succeed_sorted[0]

            # VV: If it's a repeating engine it might fail, but we expect that Source engines never fail
            offset = backend_delay_offset if top_prod.is_repeat else 0.0

            remaining_time = offset + eff_rep_int * factor - top_succeed_time

            # VV: TODO At this point we could predict WHICH of the producers will actually execute for long
            #     enough so that we reach the desired eff_rep_int (using the execute models)

            next_repeat = remaining_time
            self.log.debug("%s max producer is %s with %.2f remaining (progress:%.2f), "
                          "repeat-interval=%.2f (eri: %.2f*%.2f)" % (
                              desc.reference, top_prod.reference,
                              remaining_time, top_succeed_time,
                              next_repeat, eff_rep_int, factor
            ))

            return next_repeat + current_delay, eff_rep_int
        elif current_progress >= eff_rep_int * factor:
            # VV: looks like we should be already running, make it so we start immediately-ish
            self.log.debug("%s should be already repeating. Progress: %.2f, eri: %.2f*%.2f" % (
                desc.reference, current_progress, eff_rep_int, factor,
            ))

            # VV: If we've already thought of some repeat-interval then just use that one
            if decision.next_repeat_interval() is not None:
                return decision.next_repeat_interval(), eff_rep_int
            # VV: If this is the first time we are deciding the repeat interval then set it equal to
            #     the time that it would have been ideal to start executing
            return current_delay - (current_progress - eff_rep_int * factor), eff_rep_int
        else:
            assert not prod_running
            # VV: Producers are not running so it would be OK to assume that we can't run either
            #     but let's just pick a huge repeat-interval if nothing changes till that time we'll
            #     just do a bit of exploring in about 4 hours from the last time it executed
            #     or 10 * repeat_interval whichever comes further in the future. See comment
            #     in _guess_effective_repeat_interval for more information on why this is not a bad idea
            next_repeat = current_delay + max(4 * 3600, 10 * desc.repeat_interval)
            self.log.debug("%s producers are not running. repeat-interval:%.2f, eri: %.2f*%.2f" % (
                desc.reference, next_repeat, eff_rep_int, factor
            ))
            return next_repeat, eff_rep_int

    def _optimize_kernel(self, date_now):
        for desc in self.active_ordered:
            if desc.is_active and desc.is_repeat:
                self.optimize_single(desc, date_now)

    def optimize_single(self, desc, date_now):
        # type: (ComponentDescriptor, datetime.datetime) -> Optional[float]
        with self.lock:
            metadata = self.engine_metadata[desc]

            decision = self.decisions[desc]

            if metadata.last_run_stopped is None or decision.target_effective_repeat_interval is None:
                self.log.debug("Cannot decide because last_run_stopped: %s "
                              "and target_effective_repeat_interval %s "
                              "for %s.%s" %(
                        metadata.last_run_stopped, decision.target_effective_repeat_interval, desc.stage, desc.component
                ))
                return None

            # if producers_dirty or decision.is_dirty:
            if decision.should_exploit(metadata):
                next_repeat_interval, eri = self._exploit_desc(desc, date_now,
                                                               decision.exploit_target_factor())
            else:
                next_repeat_interval, eri = self._explore_desc(desc, date_now,
                                                               decision.explore_target_factor())

            self.log.debug("Came up with decision %s for %s.%s" % (
                next_repeat_interval, desc.stage, desc.component
            ))

            if next_repeat_interval is not None:
                decision.register_choice(metadata.last_run_stopped
                                         + datetime.timedelta(seconds=next_repeat_interval),
                                         metadata.last_run_stopped)

        return decision.next_repeat_interval()

    def suggest_next_delay(self, stage, component, date_now):
        # type: (str, str, datetime.datetime) -> Optional[float]
        with self.lock:
            try:
                desc = self.modelist.descriptors[self.experiment_name][stage][component]
            except KeyError as e:
                # VV: Haven't had the chance to digest the event message yet, this is not an error
                self.log.warning("I just got asked to Optimize an unknown Engine %s.%s" % (
                                 stage, component
                             ))
                return None

            return self.optimize_single(desc, date_now)


import math
import os

import cntk as C
import numpy as np
import skimage.transform
from PIL import Image
from pathlib import Path

def load_image_as_hwc(path, output_shape, scale=None, rotation=None, shift=None, order=1, mode='edge', normalize=True):
    image = Image.open(path)

    if not len(image.getbands()) == 3:
        image = image.convert('RGB')

    source_width = image.size[0]
    source_height = image.size[1]
    target_width = output_shape[0]
    target_height = output_shape[1]

    if source_width == target_width and source_height == target_height and scale is None and rotation is None and shift is None:
        return np.asarray(image)

    source_ratio = source_width / source_height
    target_ratio = target_width / target_height

    if target_ratio < source_ratio:
        resize_scale = target_width / source_width
    else:
        resize_scale = target_height / source_height

    if scale:
        resize_scale *= scale

    image = image.resize(size=(int(source_width * resize_scale),
                               int(source_height * resize_scale)), resample=Image.LANCZOS)

    image_array = np.asarray(image, dtype=np.float64)  # HWC

    # centerize
    t = skimage.transform.AffineTransform(
        translation=(-image.size[0] * 0.5, -image.size[1] * 0.5))

    if rotation:
        radian = (rotation / 180.0) * math.pi
        t += skimage.transform.AffineTransform(rotation=radian)

    t += skimage.transform.AffineTransform(
        translation=(target_width * 0.5, target_height * 0.5))

    if shift:
        t += skimage.transform.AffineTransform(
            translation=(target_width * shift[0], target_height * shift[1]))

    warp_shape = (output_shape[1], output_shape[0])

    image_array = skimage.transform.warp(
        image_array, (t).inverse, output_shape=warp_shape, order=order, mode=mode)

    if normalize:
        image_array /= 255.0

    return image_array

def load_model_and_tags(project_path):
    if not os.path.exists(project_path):
        raise Exception(f'{project_path} is not exists.')

    tags_path = os.path.join(project_path, 'tags.txt')
    model_path = os.path.join(project_path, 'model.cntk')

    with open(tags_path, 'r') as tags_stream:
        tag_array = np.array([tag for tag in (tag.strip()
                                              for tag in tags_stream) if tag])

    model = C.load_model(model_path)

    return (model, tag_array)

def get_files_recursively(folder_path):
    allowed_patterns = [
        '*.[Pp][Nn][Gg]',
        '*.[Jj][Pp][Gg]',
        '*.[Jj][Pp][Ee][Gg]',
        '*.[Gg][Ii][Ff]'
    ]

    image_path_list = [
        str(path) for pattern in allowed_patterns for path in Path(folder_path).rglob(pattern)]

    return image_path_list