#!/usr/bin/python
import random
import copy
from collections import defaultdict

class DP(object):
    '''Class for Davis-Putnam solver.
    '''

    def __init__(self, file):
        self.file = file
        self.split = 0
        self.count = 0

    def split_choice(self, clauses):
        '''Select a heuristic to split.

        Arguments:
            clauses {list} -- list of clauses

        Returns:
            int -- variable to split
        '''

        if self.split == 1:
            return self.random_split(clauses)
        elif self.split == 2:
            return self.JW_det_split(clauses)
        elif self.split == 3:
            return self.JW_prob_split(clauses)
        elif self.split == 4:
            return self.DLIS_det_split(clauses)
        elif self.split == 5:
            return self.DLIS_prob_split(clauses)

    def solver(self, clauses):
        '''Main method of solver.

        Arguments:
            clauses {list} -- list of clauses

        Returns:
            dict -- dictionary of variables with assigned values
        '''

        # print('|START count:', self.count)
        # print('start:', clauses)
        clauses = self.pure_literals(clauses)
        # print('after pure:', clauses)
        clauses = self.unit_clauses(clauses)
        # print('after unit:', clauses)
        if [] in clauses:
            # print('false')
            return False
        if len(clauses) == 0:
            # print('success!')
            return self.vars
        split_var = self.split_choice(clauses)
        self.count += 1
        # print(split_var)
        tmp = copy.deepcopy(clauses)
        assignment = self.solver(self.remove_clauses(split_var, clauses))
        if assignment is False:
            # print('backtracking...')
            self.count += 1
            clauses = copy.deepcopy(tmp)
            assignment = self.solver(self.remove_clauses(-split_var, clauses))
        if assignment is False:
            return False
        return self.vars

    def random_split(self, clauses):
        '''Random splitting heuristic.

        Arguments:
            clauses {list} -- list of clauses

        Returns:
            int -- variable to split
        '''

        clause = random.choice(clauses)
        split = random.choice(clause)
        return split

    def remove_clauses(self, variable, clauses):
        '''Update the list of clauses with the given variable assigned.

        Arguments:
            variable {int} -- variable with assgined value
            clauses {list} -- list of clauses

        Returns:
            list -- updated list of clauses
        '''

        new_clauses = []
        if variable >= 0:
            self.vars[variable] = True
        else:
            self.vars[abs(variable)] = False
        for clause in clauses:
            if variable in clause:
                continue
            else:
                if -variable in clause:
                    clause.remove(-variable)
                new_clauses.append(clause)
        return new_clauses

    def remove_clauses_testing_only(self, variable, clauses):
        '''Update the list of clauses with the given variable assigned.

        Arguments:
            variable {int} -- variable with assgined value
            clauses {list} -- list of clauses

        Returns:
            list -- updated list of clauses
        '''

        print('variable:', variable)
        print('number of clauses:', len(clauses))
        new_clauses = []
        if variable >= 0:
            self.vars[variable] = True
        else:
            self.vars[abs(variable)] = False
        for clause in clauses:
            if variable in clause:
                continue
            else:
                if -variable in clause:
                    clause.remove(-variable)
                new_clauses.append(clause)
        return new_clauses

    def read(self):
        '''Method for reading the clauses from the input file.

        Returns:
            list -- list of clauses
        '''

        # Initialize clauses list.
        clauses = []

        # Initialize variables.
        vars_tmp = set()

        # Start reading from the file.
        with open(self.file, 'r') as input_file:
            for line in input_file:
                parsed = line.split()

                # Check whether it is valid line or supplementary line.
                if not parsed or parsed[0] == 'p' or parsed[0] == 'c':
                    continue
                else:
                    eff_parsed = parsed[:-1]
                    clause = set()
                    for lit in eff_parsed:
                        lit = int(lit)
                        clause.add(lit)

                        # Collect variable.
                        abs_lit = abs(lit)
                        vars_tmp.add(abs_lit)
                    clauses.append(list(clause))

        # Initialize all collected variables, e.g. {'115': [False] ...} - where [truth_val]
        self.vars = dict.fromkeys(vars_tmp, False)
        return clauses

    def tautology(self, clauses):
        '''Check and remove tautology from the list of clauses.

        Returns:
            list -- list of clauses
        '''

        new_clauses = []
        check = 1
        for clause in clauses:
            for lit in clause:
                if -lit in clause:
                    check = 0
                    break
            if check == 1:
                new_clauses.append(clause)
            else:
                check = 1
        return new_clauses

    def pure_literals(self, clauses):
        '''Collect and remove the pure literals from the list of clauses.

        Returns:
            list -- list of clauses
        '''

        p_lits = set()
        non_p_lits = set()
        for clause in clauses:
            for lit in clause:
                neg_lit = -lit
                abs_lit = abs(lit)
                if neg_lit not in p_lits:
                    if abs_lit not in non_p_lits:
                        p_lits.add(lit)
                else:
                    p_lits.remove(neg_lit)
                    non_p_lits.add(abs_lit)
        for lit in p_lits:
            clauses = self.remove_clauses(lit, clauses)
        return clauses

    def unit_clauses(self, clauses):
        '''Collect and remove unit clauses from the list of clauses.

        Returns:
            list -- list of clauses
        '''

        unit_var = set()
        for clause in clauses:
            if len(clause) == 1:
                unit_var.add(clause[0])
        while len(unit_var) > 0:
            for unit in unit_var:
                clauses = self.remove_clauses(unit, clauses)
            unit_var = set()
            clauses = self.unit_clauses(clauses)
        return clauses

    def JW_det_split(self, clauses):
        '''Use deterministic Jeroslow-Wang heuristic to split variable.

        Arguments:
            clauses {list} -- list of clauses

        Returns:
            int -- selected variable to split
        '''

        J = defaultdict(int)
        for clause in clauses:
            clause_len = len(clause)
            for lit in clause:
                J[lit] += 2 ** (-clause_len)

        max_pair = [-1, 0]
        for k in J.keys():
            sum_J = J[k] + J[-k]
            if sum_J > max_pair[1] and max_pair[0] != -k:
                max_pair = [k, sum_J]

        split = abs(max_pair[0])
        if J[split] >= J[-split]:
            return split
        else:
            return -split

    def JW_prob_split(self, clauses):
        '''Use probabilistic Jeroslow-Wang heuristic to split variable.

        Arguments:
            clauses {list} -- list of clauses

        Returns:
            int -- selected variable to split
        '''

        J = defaultdict(int)
        for clause in clauses:
            clause_len = len(clause)
            for lit in clause:
                J[lit] += 2 ** (-clause_len)

        choices = []
        vals = []
        for k, v in J.items():
            lit = abs(k)
            if lit not in choices:
                choices.append(lit)
                vals.append(J[k] + J[-k])

        split = random.choices(choices, weights=vals, k=1)
        split = split[0]

        split = random.choices([split, -split], weights=[J[split], J[-split]], k=1)
        split = split[0]
        return split

    def DLIS_det_split(self, clauses):
        '''Use deterministic DLIS heuristic to split variable.

        Arguments:
            clauses {list} -- list of clauses

        Returns:
            int -- selected variable to split
        '''

        C_p = defaultdict(int)
        C_n = defaultdict(int)
        for clause in clauses:
            for lit in clause:
                if lit > 0:
                    C_p[lit] += 1
                else:
                    C_n[lit] += 1
        max_pair_p = [0, -1]
        for k, v in C_p.items():
            if v > max_pair_p[1]:
                max_pair_p = [k, v]

        max_pair_n = [0, -1]
        for k, v in C_n.items():
            if v > max_pair_n[1]:
                max_pair_n = [k, v]

        if max_pair_p[1] > max_pair_n[1]:
            return max_pair_p[0]
        else:
            return max_pair_n[0]

    def DLIS_prob_split(self, clauses):
        '''Use probabilistic DLIS heuristic to split variable.

        Arguments:
            clauses {list} -- list of clauses

        Returns:
            int -- selected variable to split
        '''

        C_p = defaultdict(int)
        C_n = defaultdict(int)
        for clause in clauses:
            for lit in clause:
                if lit > 0:
                    C_p[lit] += 1
                else:
                    C_n[lit] += 1

        choices_p = []
        weights_p = []
        for k, v in C_p.items():
            choices_p.append(k)
            weights_p.append(v)

        choices_n = []
        weights_n = []
        for k, v in C_n.items():
            choices_n.append(k)
            weights_n.append(v)

        choice_p = random.choices(choices_p, weights=weights_p, k=1)
        choice_p = choice_p[0]
        val_p = C_p[choice_p]

        choice_n = random.choices(choices_n, weights=weights_n, k=1)
        choice_n = choice_n[0]
        val_n = C_n[choice_n]

        split = random.choices([choice_p, choice_n], weights=[val_p, val_n], k=1)
        split = split[0]
        return split

    def output_results(self, vars=None):
        '''Method for outputting the results into input_filename.out.

        Keyword Arguments:
            vars {dict} -- dictionary to store the variables with their assignments (default: {None})
        '''

        input_file_name = self.file.split('/')
        input_file_name = input_file_name[-1]
        output_file = input_file_name[:-4] + '.out'
        if vars is None:
            with open(output_file, 'w') as output:
                output.write('')
        else:
            assignments = []
            for k, v in vars.items():
                if v:
                    assignments.append(str(k) + ' 0\n')
                else:
                    assignments.append(str(-k) + ' 0\n')
            with open(output_file, 'w') as output:
                output.write('p cnf {} {}\n'.format(len(vars), len(vars)))
                output.writelines(assignments)


#add all the digits of a positive integer number

def sum_of_digits(n):
    suma=0
    while n>0:
        suma=suma+n%10
        n=(n-n%10)/10
    return suma

def sum_rec(n):
    assert n>=0 and int(n)==n, "enter a positive integer only!"
    if n==0:
        return 0
    else:
        return n%10 + sum_rec(n//10)


#!/usr/bin/env python3
import argparse
import collections
import re
import sys

parser = argparse.ArgumentParser()
parser.add_argument("train_data", type=str, help="Training data file")
args = parser.parse_args()

lemma_strip_re = re.compile(r"(?<=.)(?:`|_|-[^0-9]).*$")
def lemma_tag_key(lemma, tag):
    return lemma_strip_re.sub("", lemma) + "\t" + tag

train = collections.defaultdict(lambda: dict())
with open(args.train_data, mode="r", encoding="utf-8") as train_file:
    for line in train_file:
        line = line.rstrip("\n")
        if not line: continue

        form, lemma, tag, *_ = line.split("\t")
        train[form][lemma_tag_key(lemma, tag)] = (lemma, tag)

for line in sys.stdin:
    line = line.rstrip("\n")

    if not line:
        print()
    else:
        form, lemma, tag, *analyses = line.split("\t")
        assert len(analyses) % 2 == 0

        if len(analyses):
            train_analyses = list(train.get(form, {}).items()) \
                + list(train.get(form[0] + form[1:].lower(), {}).items()) \
                + list(train.get(form.lower(), {}).items())
            if train_analyses:
                analyses_set = set(lemma_tag_key(analyses[i], analyses[i + 1]) for i in range(0, len(analyses), 2))
                for key, lemma_tag in train_analyses:
                    if key not in analyses_set:
                        analyses.extend(lemma_tag)
                        analyses_set.add(key)

        print("\t".join([form, lemma, tag] + analyses))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
Prepare dataset for bayesian variable selection.
"""

import argparse
import logging

import pandas as pd
from sklearn.pipeline import Pipeline

from src.common.df_csv_writing import write_df_to_csv
from src.common.json_logging import setup_logging
from src.data.imputations import impute
from src.data.imputations.impute import DataImputer
from src.data.na_thresholding.transformer import ThresholdingMissingDataColumnDropper, ValuePresenceRowFilter

setup_logging("src/common/logging.json")  # setup logger
logger = logging.getLogger(__name__)

if __name__ == "__main__":

    # configure parser and parse arguments
    parser = argparse.ArgumentParser(description='Prepare dataset for bayesian variable selection.')
    parser.add_argument('--dataset', type=str, help='The path to the dataset file', required=True)
    parser.add_argument('--csv_separator', type=str, default=',', help='The separator of the data columns')
    parser.add_argument('--diagnosis_indicator_column', action='append', help='Typical measurements done when evaluating the plausibility of a certain diagnosis. Repeat flag to provide multiple columns')
    parser.add_argument('--skip_na_drop_threshold', action='store_true', help='Skip drop thresholding')
    parser.add_argument('--na_drop_threshold', type=float, default=1.0, help='Minimum amount of NA in column for it to be dropped [0;1]')
    parser.add_argument('--skip_imputation', action='store_true', help='Skip imputation')
    parser.add_argument('--imputation_type', type=str, default='ITERATIVE', help='The type of imputation to perform')
    parser.add_argument('--niter', type=int, default=1000, help='The number of iterations to perform, in case an interative method is chosen')

    args = parser.parse_args()
    dataset_path = args.dataset
    csv_separator = args.csv_separator
    indicator_columns = args.diagnosis_indicator_column
    do_na_drop_threshold = not args.skip_na_drop_threshold
    na_drop_threshold = args.na_drop_threshold
    do_imputation = not args.skip_imputation
    imputation_type = impute.ImputationType[args.imputation_type]
    iteration_qty = args.niter

    logger.info(str({"message": "PREPARE NEW DATASET",
                     "path": dataset_path,
                     "imputation_type": imputation_type,
                     "iteration_qty": iteration_qty})
                )

    file_appendix = ''  # prepare file appendix telling us what has been done

    logger.info(str({"message": "Load dataset",
                     "path": dataset_path}))
    df = pd.read_csv(dataset_path, header=0, sep=csv_separator)  # read data from csv

    # prepare pipeline and run it
    components = []
    if indicator_columns is not [] and indicator_columns is not None:
        components.append(('filter_by_indicator_columns', ValuePresenceRowFilter(required_columns=indicator_columns)))
    file_appendix += '_indicatorCols_' + str(indicator_columns)

    if do_na_drop_threshold:
        components.append(('drop_above_threshold_na_columns', ThresholdingMissingDataColumnDropper(na_drop_threshold=na_drop_threshold)))
        file_appendix += '_naLimit_' + str(na_drop_threshold)

    if do_imputation:
        components.append(('impute_missing_values', DataImputer(imputation_type=imputation_type, iteration_qty=iteration_qty)))
        file_appendix += '_impType_' + imputation_type.name + '_nIter_' + str(iteration_qty)

    pipeline = Pipeline(components)
    df = pipeline.fit_transform(df)  # apply pipeline

    logger.info({"message": "Results",
                 "dataset dimensions":
                 df.shape})

    # write dataset to file
    write_df_to_csv(df=df, store_path='data/interim/', initial_path=dataset_path, file_appendix=file_appendix)


"""Adding mathematical fractions"""

from helpers.task import TaskBase, task_to_list, get_input

class Task(TaskBase):
    """Adding mathematical fractions"""

    tasklist: list = []

    @staticmethod
    def gcd(num_a: int, num_b: int) -> int:
        """Greatest Common Divisor"""
        if num_a == 0:
            return num_b
        return Task.gcd(num_b % num_a, num_a)

    @task_to_list(tasklist)
    def task1(self) -> dict[str, int]:
        """Write a recursive function, GCD(a,b), that evaluates
        the Greatest Common Divisor (i.e., the largest common factor)
        between two numbers, a and b."""
        num_a: int = get_input(int, "Enter the first number")
        num_b: int = get_input(int, "Enter the second number")
        return {"gcd": Task.gcd(num_a, num_b)}

    @staticmethod
    def lcm(num_a: int, num_b: int) -> int:
        """Least Common Multiple"""
        return (num_a * num_b) // Task.gcd(num_a, num_b)

    @task_to_list(tasklist)
    def task2(self) -> dict[str, int]:
        """Write a function, LCM(a,b), that evaluates
        the Least Common Multiple (i.e., the largest common factor)
        between two numbers, a and b."""
        num_a: int = get_input(int, "Enter the first number")
        num_b: int = get_input(int, "Enter the second number")
        return {"lcm": Task.lcm(num_a, num_b)}

    @staticmethod
    def add_fractions(list_n: list[int], list_d: list[int]) -> tuple[int, int]:
        """Add fractions"""
        num: int = 0
        denom: int = 1
        for i, val in enumerate(list_d):
            num *= val
            num += denom * list_n[i]
            denom *= val
        gcd: int = Task.gcd(num, denom)
        return num // gcd, denom // gcd

    @task_to_list(tasklist)
    def task3(self) -> dict[str, int]:
        """Write a function, AddFractions(N,D), that receives two lists of
        integer values, N and D, representing the numerators and denominators
        of some fractions, respectively, and returns two values, Nt and Dt.
        Ensure that the final fraction, Nt/Dt, is at its minimal form."""
        str_n: str = get_input(str, "Enter numerators separated by commas")
        str_d: str = get_input(str, "Enter denominators separated by commas")
        list_n: list[int] = [int(i) for i in str_n.split(",")]
        list_d: list[int] = [int(i) for i in str_d.split(",")]
        num: int
        denom: int
        num, denom = Task.add_fractions(list_n, list_d)
        return {"nt": num, "dt": denom}

if __name__ == "__main__":
    task: Task = Task("D")
    task.run_tasks()
    quiz_n: list[int] = [3, 1, 3, 2, 5, 7, 2, 4, 6, 1, 7, 4]
    quiz_d: list[int] = [4, 5, 8, 16, 2, 4, 32, 2, 4, 3, 36, 45]
    print(Task.add_fractions(quiz_n, quiz_d))


from typing import Type, Any, Callable, Union, List, Optional
import tensorflow as tf
import tensorflow.keras.layers
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Conv2D, ReLU, BatchNormalization, Dropout, AveragePooling2D
from utils import drop_path
from operations import *
from utils import SeparableConv2d

def conv3x3(in_planes: int, out_planes: int, stride: int = 1, groups: int = 1, dilation: int = 1) -> Conv2D:
    """3x3 convolution with padding"""
    if dilation == 1:
        padding = "same"
    else:
        padding = "valid"
    return Conv2D(out_planes, kernel_size=(3,3), strides=(stride, stride), padding=padding, groups=groups, use_bias=False, dilation_rate=dilation)
    # return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
    #                  padding=dilation, groups=groups, bias=False, dilation=dilation)

def sep_conv3x3(in_planes: int, out_planes: int, stride: int = 1, groups: int = 1, dilation: int = 1) -> SeparableConv2d:
    """3x3 convolution with padding"""
    if dilation == 1:
        padding = "same"
    else:
        padding = "valid"
    return SeparableConv2d(in_planes, out_planes, kernel_size=3, stride=stride,
                     padding=padding, bias=False, dilation=dilation)

def conv1x1(in_planes: int, out_planes: int, stride: int = 1) -> Conv2D:
    """1x1 convolution"""
    return Conv2D(out_planes, kernel_size=(1,1), strides=(stride, stride), use_bias=False)
    # return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)

def _bn_function_factory(norm, relu, conv):
    def bn_function(*inputs):
        #TODO
        concated_features = tf.concat(inputs, 1)
        # concated_features = torch.cat(inputs, 1)
        bottleneck_output = conv(relu(norm(concated_features)))
        return bottleneck_output

    return bn_function

class BasicBlock(tf.Module):
    expansion: int = 1

    def __init__(
        self,
        inplanes: int,
        planes: int,
        stride: int = 1,
        downsample: Optional[tf.Module] = None,
        groups: int = 1,
        base_width: int = 64,
        dilation: int = 1,
        norm_layer: Optional[Callable[..., tf.Module]] = None,
        separable: bool = True,
    ) -> None:
        super(BasicBlock, self).__init__()
        if norm_layer is None:
            norm_layer = BatchNormalization
        #if groups != 1 or base_width != 64:
        if groups != 1: # or base_width != 64:
            raise ValueError('BasicBlock only supports groups=1 and base_width=64')
        if dilation > 1:
            raise NotImplementedError("Dilation > 1 not supported in BasicBlock")
        # Both self.conv1 and self.downsample layers downsample the input when stride != 1
        if separable:
            self.conv1 = sep_conv3x3(inplanes, planes, stride)  #conv3x3(inplanes, planes, stride)
            self.conv2 = sep_conv3x3(planes, planes) #conv3x3(planes, planes)
        else:
            self.conv1 = conv3x3(inplanes, planes, stride)
            self.conv2 = conv3x3(planes, planes)

        self.bn1 = norm_layer()
        self.relu = ReLU()
        self.bn2 = norm_layer()
        self.downsample = downsample
        self.stride = stride

    def __call__(self, x: tf.Variable) -> tf.Variable:
        identity = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)

        if self.downsample is not None:
            identity = self.downsample(x)

        out += identity
        out = self.relu(out)

        return out

class Bottleneck(tf.Module):
    # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2)
    # while original implementation places the stride at the first 1x1 convolution(self.conv1)
    # according to "Deep residual learning for image recognition"https://arxiv.org/abs/1512.03385.
    # This variant is also known as ResNet V1.5 and improves accuracy according to
    # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch.

    def __init__(
        self,
        inplanes: int,
        planes: int,
        stride: int = 1,
        downsample: Optional[tf.Module] = None,
        groups: int = 1,
        base_width: int = 64,
        dilation: int = 1,
        norm_layer: Optional[Callable[..., tf.Module]] = None,
        separable: bool = True,
        expansion: int = 4,
    ) -> None:
        super(Bottleneck, self).__init__()
        if norm_layer is None:
            norm_layer = BatchNormalization
        width = int(planes * (base_width / 64.)) * groups
        # Both self.conv2 and self.downsample layers downsample the input when stride != 1
        self.conv1 = conv1x1(inplanes, width)
        self.bn1 = norm_layer()
        # self.bn1 = norm_layer(width)
        self.conv2 = conv3x3(width, width, stride, groups, dilation)
        self.bn2 = norm_layer()
        # self.bn2 = norm_layer(width)
        self.conv3 = conv1x1(width, planes * expansion)
        self.bn3 = norm_layer()
        # self.bn3 = norm_layer(planes * expansion)
        self.relu = ReLU()
        self.downsample = downsample
        self.stride = stride

    def __call__(self, x: tf.Variable) -> tf.Variable:
        identity = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)
        out = self.relu(out)

        out = self.conv3(out)
        out = self.bn3(out)

        if self.downsample is not None:
            identity = self.downsample(x)

        out += identity
        out = self.relu(out)

        return out

class _DenseLayer(tf.Module):
    def __init__(self, num_input_features, growth_rate, bn_size, drop_rate, efficient=False):
        super(_DenseLayer, self).__init__()
        self.add_module('norm1', BatchNormalization(num_input_features)),
        self.add_module('relu1', ReLU()),
        self.add_module('conv1', Conv2D(bn_size * growth_rate,
                        kernel_size=(1,1), strides=(1,1), use_bias=False)),
        self.add_module('norm2', BatchNormalization(bn_size * growth_rate)),
        self.add_module('relu2', ReLU()),
        self.add_module('conv2', Conv2D(growth_rate,
                        kernel_size=(3,3), strides=(1,1), padding="same", use_bias=False)),
        self.drop_rate = drop_rate
        self.efficient = efficient

    def __call__(self, *prev_features):
        bn_function = _bn_function_factory(self.norm1, self.relu1, self.conv1)
        if self.efficient and any(prev_feature.requires_grad for prev_feature in prev_features):
            bottleneck_output = tf.keras.Model.save_weights.checkpoint(bn_function, *prev_features)
        else:
            bottleneck_output = bn_function(*prev_features)
        new_features = self.conv2(self.relu2(self.norm2(bottleneck_output)))
        if self.drop_rate > 0:
            new_features = Dropout(new_features, p=self.drop_rate, training=self.training)
        return new_features

class BasicDenseLayer(tf.Module):
    def __init__(self, planes, bn_size):
        super(BasicDenseLayer, self).__init__()
        self.add_module('norm1', BatchNormalization(planes)),
        self.add_module('relu1', ReLU()),
        self.add_module('conv1', Conv2D(bn_size * planes,
                        kernel_size=(1,1), strides=(1,1), use_bias=False)),
        self.add_module('norm2', BatchNormalization(bn_size * planes)),
        self.add_module('relu2', ReLU()),
        self.add_module('conv2', Conv2D(planes,
                        kernel_size=(3,3), strides=(1,1), padding="same", use_bias=False)),

    def __call__(self, *prev_features):
        bn_function = _bn_function_factory(self.norm1, self.relu1, self.conv1)
        bottleneck_output = bn_function(*prev_features)
        new_features = self.conv2(self.relu2(self.norm2(bottleneck_output)))
        return new_features

class Transition(Sequential):
    def __init__(self, num_input_features, num_output_features):
        super(Transition, self).__init__()
        self.add_module('norm', BatchNormalization(num_input_features))
        self.add_module('relu', ReLU())
        self.add_module('conv', Conv2D(num_output_features,
                                          kernel_size=(1,1), strides=(1,1), use_bias=False))
        self.add_module('pool', AveragePooling2D(pool_size=(2,2), strides=2))

class DenseBlock(tf.Module):
    def __init__(self, num_layers, num_input_features, bn_size, growth_rate, drop_rate, efficient=False):
        super(DenseBlock, self).__init__()
        for i in range(num_layers):
            layer = _DenseLayer(
                num_input_features + i * growth_rate,
                growth_rate=growth_rate,
                bn_size=bn_size,
                drop_rate=drop_rate,
                efficient=efficient,
            )
            self.add_module('denselayer%d' % (i + 1), layer)

    def __call__(self, init_features):
        features = [init_features]
        for name, layer in self.named_children():
            new_features = layer(*features)
            features.append(new_features)
        return tf.concat(features, 1)

class Cell(tf.Module):

  def __init__(self, genotype, C_prev_prev, C_prev, C, reduction, reduction_prev):
    super(Cell, self).__init__()
    print(C_prev_prev, C_prev, C)

    if reduction_prev:
      self.preprocess0 = FactorizedReduce(C_prev_prev, C)
    else:
      self.preprocess0 = ReLUConvBN(C_prev_prev, C, 1, 1, 0)
    self.preprocess1 = ReLUConvBN(C_prev, C, 1, 1, 0)

    if reduction:
      op_names, indices = zip(*genotype.reduce)
      concat = genotype.reduce_concat
    else:
      op_names, indices = zip(*genotype.normal)
      concat = genotype.normal_concat
    self._compile(C, op_names, indices, concat, reduction)

  def _compile(self, C, op_names, indices, concat, reduction):
    assert len(op_names) == len(indices)
    self._steps = len(op_names) // 2
    self._concat = concat
    self.multiplier = len(concat)
    self._ops = tf.Module()
    # self._ops = nn.ModuleList()
    for name, index in zip(op_names, indices):
      stride = 2 if reduction and index < 2 else 1
      op = OPS[name](C, stride, True)
      self._ops += [op]
    self._indices = indices

  def __call__(self, s0, s1, drop_prob):
    s0 = self.preprocess0(s0)
    s1 = self.preprocess1(s1)

    states = [s0, s1]
    for i in range(self._steps):
      h1 = states[self._indices[2*i]]
      h2 = states[self._indices[2*i+1]]
      op1 = self._ops[2*i]
      op2 = self._ops[2*i+1]
      h1 = op1(h1)
      h2 = op2(h2)
      if self.training and drop_prob > 0.:
        if not isinstance(op1, Identity):
          h1 = drop_path(h1, drop_prob)
        if not isinstance(op2, Identity):
          h2 = drop_path(h2, drop_prob)
      s = h1 + h2
      states += [s]
    return tf.concat([states[i] for i in self._concat], dim=1)


from logging import getLogger

import reversion

from datahub.dbmaintenance.management.base import CSVBaseCommand
from datahub.dbmaintenance.utils import parse_bool, parse_uuid, parse_uuid_list
from datahub.investment.project.models import InvestmentProject

logger = getLogger(__name__)

class Command(CSVBaseCommand):
    """
    Command to update investment_project.uk_region_locations and
    investment_project.allow_blank_possible_uk_regions.
    """

    def add_arguments(self, parser):
        """Define extra arguments."""
        super().add_arguments(parser)

        parser.add_argument(
            '--ignore-old-regions',
            action='store_true',
            default=False,
            help='If True it does not check if the regions match the old_uk_region_locations '
                 'column.',
        )

    def _process_row(self, row, simulate=False, ignore_old_regions=False, **options):
        """Process one single row."""
        pk = parse_uuid(row['id'])
        investment_project = InvestmentProject.objects.get(pk=pk)
        allow_blank_possible_uk_regions = parse_bool(row['allow_blank_possible_uk_regions'])
        uk_region_locations = parse_uuid_list(row['uk_region_locations'])

        current_regions = investment_project.uk_region_locations.all()
        current_region_ids = set(region.pk for region in current_regions)
        if (investment_project.allow_blank_possible_uk_regions == allow_blank_possible_uk_regions
                and current_region_ids == set(uk_region_locations)):
            return

        if not ignore_old_regions:
            old_uk_region_locations = parse_uuid_list(row['old_uk_region_locations'])

            if current_region_ids != set(old_uk_region_locations):
                return

        investment_project.allow_blank_possible_uk_regions = allow_blank_possible_uk_regions

        if simulate:
            return

        with reversion.create_revision():
            investment_project.save(
                update_fields=('allow_blank_possible_uk_regions',),
            )
            investment_project.uk_region_locations.set(uk_region_locations)
            reversion.set_comment('Possible UK regions data migration correction.')


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf
from tensorflow_TB.lib.trainers.trainer_base import TrainerBase
from math import ceil
from tensorflow_TB.lib.base.collections import TRAIN_SUMMARIES
import numpy as np
from tensorflow_TB.utils.misc import collect_features

class ClassificationTrainer(TrainerBase):
    """Implementing classification trainer
    Using the entire labeled trainset for training"""

    def train_step(self):
        '''Implementing one training step'''
        _ , images, labels = self.dataset.get_mini_batch('train', self.plain_sess)
        _ , self.global_step = self.sess.run([self.model.train_op, self.model.global_step],
                                              feed_dict={self.model.images: images,
                                                         self.model.labels: labels,
                                                         self.model.is_training: True})

    def eval_step(self):
        '''Implementing one evaluation step.'''
        self.log.info('start running eval within training. global_step={}'.format(self.global_step))
        (labels, predictions) = \
            collect_features(
                agent=self,
                dataset_name='validation',
                fetches=[self.model.labels, self.model.predictions],
                feed_dict={self.model.dropout_keep_prob: 1.0})
        if self.model.one_hot_labels:
            labels = np.argmax(labels, axis=1)
        score = np.average(labels == predictions)

        # sample loss/summaries for only the first batch
        (summaries, loss) = self.sample_stats(dataset_name='validation')

        self.validation_retention.add_score(score, self.global_step)
        self.tb_logger_validation.log_scalar('score', score, self.global_step)
        self.tb_logger_validation.log_scalar('best score', self.validation_retention.get_best_score(), self.global_step)
        self.summary_writer_validation.add_summary(summaries, self.global_step)
        self.summary_writer_validation.flush()
        self.log.info('EVALUATION (step={}): loss: {}, score: {}, best score: {}' \
                      .format(self.global_step, loss, score, self.validation_retention.get_best_score()))

    def test_step(self):
        '''Implementing one test step.'''
        self.log.info('start running test within training. global_step={}'.format(self.global_step))
        (labels, predictions) = \
            collect_features(
                agent=self,
                dataset_name='test',
                fetches=[self.model.labels, self.model.predictions],
                feed_dict={self.model.dropout_keep_prob: 1.0})
        if self.model.one_hot_labels:
            labels = np.argmax(labels, axis=1)
        score = np.average(labels == predictions)

        # sample loss/summaries for only the first batch
        (summaries, loss) = self.sample_stats(dataset_name='test')

        self.test_retention.add_score(score, self.global_step)
        self.tb_logger_test.log_scalar('score', score, self.global_step)
        self.tb_logger_test.log_scalar('best score', self.test_retention.get_best_score(), self.global_step)
        self.summary_writer_test.add_summary(summaries, self.global_step)
        self.summary_writer_test.flush()
        self.log.info('TEST (step={}): loss: {}, score: {}, best score: {}' \
                      .format(self.global_step, loss, score, self.test_retention.get_best_score()))

    def sample_stats(self, dataset_name):
        """Sampling validation/test summary and loss only for one eval batch."""
        if dataset_name == 'validation':
            self.plain_sess.run(self.dataset.validation_iterator.initializer)
        elif dataset_name == 'test':
            self.plain_sess.run(self.dataset.test_iterator.initializer)
        else:
            err_str = 'sample_stats must be called only with dataset=validation/test, not {}'.format(dataset_name)
            self.log.error(err_str)
            raise AssertionError(err_str)

        _, images, labels = self.dataset.get_mini_batch(dataset_name, self.plain_sess)
        (summaries, loss) = self.plain_sess.run([self.model.summaries, self.model.cost],
                                          feed_dict={self.model.images: images,
                                                     self.model.labels: labels,
                                                     self.model.is_training: False,
                                                     self.model.dropout_keep_prob: 1.0})
        return summaries, loss

    def get_train_summaries(self):
        super(ClassificationTrainer, self).get_train_summaries()
        tf.add_to_collection(TRAIN_SUMMARIES, tf.summary.image('input_images', self.model.images))


from core.directory_utils import get_input_files
from business.pdf_service import open_pdf
from business.xp_service import extract_data_from_xp_pdf

def notas_corretagem_parser():
    files = get_input_files()
    for file in files:
        current_pdf = open_pdf(file)
        extract_data_from_xp_pdf(current_pdf)

if __name__ == "__main__":
    notas_corretagem_parser()


import numpy as np
import matplotlib.pyplot as plt
import multiprocessing as mp
import os
from time import time
import random
import math
import glob
from tqdm import tqdm
import pickle
import gc
from heiner.dataloader import DataLoader as HeinerDataloader
from myutils import printerror
from constants import *

# conclusion from profiling below:
# - inplace standardization: reduces the 0.8s per batch that the original version consumed!
# - threading (workers=1, multiprocessing=False) superior: 3.6s (2.7s without input standardization)
# - singleproc (workers=0) a bit slower: 4.4s
# - multiprocessing (workers=1, multiprocessing=True) inadequate because of slow serialization through
#                                                     queue across processes: 12.8s
# experiment2:
# inplace standardization vs original
# - original after 70 batches: 2.0s generator, 0.82s train_predict, 0.92s metrics
# - inplace after 70 batches: 1.18s generator, 0.52s train_predict, 0.96s metrics <--- first try
# - inplace after 70 batches: 0.9s generator, 0.59 train_predict, 0.86 metrics <-- second try
# - no input std after 70 batches: 0.5s generator, 0.54 train_predict, 0.95 metrics
#
# experiment 1:
# runtime profiling exploration (evaluated on sabik with batch size 128, sceneinstancebufsize 2000):
# - direct runtimes (via running this file):
#   - total time: ~2-3 sec
#   - instantiating scene instance buffers (i.e., loading file): 1...1.6 sec
#   - assignment of blocks to the batch array: 0.2 sec
#   - standardization: 0.8 sec
#   - all other parts are negligible
# - effective runtimes (via running training.py => generator_extension.py)
#   - singleprocessing (0 workers) / multiprocessing False
#     - total: ~4.4s (here/below: avg after 100 batches)
#     - generator: 2.6s (cf. above)
#     - train_predict: 0.7s
#     - metrics: 0.9s
#   - multithreading (1 worker) / multiprocessing False (same condition as others) [with standardization]
#       - total: ~3.6  (after ~200 batches)
#       - generator: 1.4s
#       - train_predict: 1.25s
#       - metrics: 0.95s
#   - multithreading (1 worker) / multiprocessing False (same condition as others) [without standardization]
#       - total: ~2.7s  (after ~1000 batches)
#       - generator: 0.8s
#       - train_predict: 0.6s
#       - metrics: 0.9s
#   - multiprocessing (1 worker) / multiprocessing True (note that the cpu was not freely available to me but similar to above/singleproc experiment)
#       - total: ~ 12.8s (here/below: avg after 60 batches)
#       - generator: ~10.0s [higher than singleproc!]
#       - train_predict: 1.25s [strange: higher than singleproc??]
#       - metrics 1.1s [strange: higher than singleproc??]

# buffer of a single scene instance (is instantiated e.g. 2000 times in a BatchLoader object in train mode)
class SceneInstanceBuffer:

    def __init__(self, filename, batchloader, mode, params):
        # #self.filename_and_or_sceneinstance_id = filename_and_or_sceneinstance_id
        self.mode = mode
        self.params = params
        self.filename = filename
        self.batchloader = batchloader

        # scene instance id
        self.scene_instance_id = self.batchloader.scene_instance_ids_dict[filename]
        # length
        self.scene_instance_length = self.batchloader.length_dict[filename]
        # positions and overlaps
        self.positions, self.overlaps = self.batchloader.block_positions_and_overlaps_dict[self.filename]

        # load data
        with np.load(filename) as data:
            # features
            self.x = data['x']
            # remark: input standardization will be done in buffer for the final batch (to be compatible with Heiner's code)

            # labels
            self.y = data['y'] if params['instantlabels'] else data['y_block']

        # saving iterator as attribute
        self.iter = self.block_iterator()

    def __len__(self):
        return len(self.positions)

    def block_iterator(self):

        empty = False
        blockid = 0
        while (not empty):

            # get current position and overlap
            position = self.positions[blockid]
            overlap = self.overlaps[blockid]

            # collect batchlength long features and labels from frame index position
            x_block = self.x[0, position:position+self.params['batchlength'], :] # ignore dummy batch dim
            y_block = self.y[0, position:position+self.params['batchlength'], :] # ignore dummy batch dim

            # adding scene instance id as second label id
            # remark: for compatibility with Heiner's accuracy utils we need to provide the scene instance id for each
            # frame although in our case all frames have the same scene instance id
            sid_block = self.scene_instance_id * np.ones_like(y_block, dtype=DTYPE_DATA)
            y_concat_sid_block = np.zeros((*y_block.shape, 2), dtype=DTYPE_DATA)
            y_concat_sid_block[:, :, 0] = y_block
            y_concat_sid_block[ :, :, 1] = sid_block

            # set mask values (overlapping frames should not be counted twice for loss and accuracy metrics)
            # remark: both the labels and the scene id need to be masked according to heiner's accuracy utils
            if overlap > 0: # except first overlap (would though be respected by following slicing as well)
                y_concat_sid_block[:overlap, :, :] = self.params['mask_value']

            yield x_block, y_concat_sid_block

            # last blockid processed:
            if blockid+1 == len(self):
                empty = True
            # next blockid:
            else:
                blockid += 1

# batchloader that internally uses a set of many (e.g. 2000) scene instance buffers to fetch blocks from
# one scene instance has between 3,000 and 20,000 frames, avg < 4,000 frames
# => avg host mem required per buffered scene instance: (160+13) * 4,000 * 4 Byte < 2.8 MB
# => a value of params['sceneinstancebufsize']=1000 implies up to 2.8 GB host mem occupancy
class BatchLoader(HeinerDataloader):
    def __init__(self, params, mode, fold_nbs, scene_nbs, batchsize, seed=1):
        label_mode = 'instant' if params['instantlabels'] else 'blockbased'
        # initializing super constructor with values from above or with their defaults (copied from super init)
        super().__init__(mode=mode, label_mode=label_mode,
                       fold_nbs=fold_nbs, scene_nbs=scene_nbs, batchsize=batchsize, timesteps=params['batchlength'], epochs=params['maxepochs'],
                       buffer=10, features=DIM_FEATURES, classes=DIM_LABELS, path_pattern=DATA_ROOT+'/',
                       seed=seed, seed_by_epoch=True, priority_queue=True, use_every_timestep=False, mask_val=params['mask_value'],
                       val_stateful=False, k_scenes_to_subsample=-1,
                       input_standardization=not params['noinputstandardization'])

        self.params = params

        self.free_memory_batches = 10 # run garbage collector after 10 batches

        # we require scene instance buffers to be larger than the batchsize (assumed in refill logic)
        assert self.params['sceneinstancebufsize'] > self.params['batchsize']

        # calculate or load filename->scene instance id, filename->scene instance length, filename->positions,overlaps dicts
        self.scene_instance_ids_dict_()
        self.length_dict_()
        self.block_positions_and_overlaps_dict = None # the above are initialized in HeinerDataloader
        self._block_positions_and_overlaps_dict()

        # get number of batches per epoch
        self._calculate_batchnumber()

        print('batchloader {} created ({} batches of size {} [per epoch] and length {} / historylength {}) using {} labels'.
              format(mode, self.batches_per_epoch, params['batchsize'], params['batchlength'], self.params['historylength'], label_mode))

        # set batch array
        self.batch_x = np.zeros((self.params['batchsize'], self.params['batchlength'], DIM_FEATURES), dtype=DTYPE_DATA)
        self.batch_y = np.zeros((self.params['batchsize'], self.params['batchlength'], DIM_LABELS, 2), dtype=DTYPE_DATA)

        # initialize state of the batch loader, will set self.epoch = 0
        self.init_epoch(first=True)

    def _calculate_batchnumber(self):
        blocks = 0
        assert len(self.filenames_all) == len(self.filenames)
        for filename in self.filenames_all:
            positions, overlaps = self.block_positions_and_overlaps_dict[filename]
            blocks += len(positions)

        self.batches_per_epoch = int(math.ceil(blocks/self.params['batchsize']))

    def _block_positions_and_overlaps_dict(self):
        if self.block_positions_and_overlaps_dict is None:
            pickle_path = os.path.join(self.pickle_path, 'block_positions_and_overlaps_dict.pickle')

            # update stored dict if current parametrization is not yet contained:
            if not os.path.exists(pickle_path):
                self._create_block_positions_and_overlaps_dict(pickle_path=pickle_path)

            # read stored dict(batchlength,historylength) of dicts(filename)
            with open(pickle_path, 'rb') as handle:
                all_block_positions_and_overlaps_dict = pickle.load(handle)

            dict_key = (self.params['batchlength'], self.params['historylength'])

            # update stored dict if current parametrization is not yet contained:
            if dict_key not in all_block_positions_and_overlaps_dict:
                self._create_block_positions_and_overlaps_dict(pickle_path=pickle_path)
                with open(pickle_path, 'rb') as handle:
                    all_block_positions_and_overlaps_dict = pickle.load(handle)

            # now we have ensured that the current parametrization exists in all_* dict
            self.block_positions_and_overlaps_dict = all_block_positions_and_overlaps_dict[dict_key]

        return self.block_positions_and_overlaps_dict

    def _create_block_positions_and_overlaps_dict(self, pickle_path):

        # calculate positions (startind frame indices) and overlaps (number of frames) of consecutive blocks
        # of a scene instance that is represented here only by its length
        # remark: the overlap with the previous block can vary though:
        #         first block (no overlap at all), intermediate blocks (overlap historysize-1) and
        #         the last block (overlap s.t. the end of the scene instance is exactly approached => quite large overlap possible)
        def calculate_block_positions_and_overlaps(scene_instance_length, batch_length, history_length):

            # assume that at least one full block (with batchlength) exists in the the scene instance
            assert batch_length <= scene_instance_length

            # first block
            block_idx = 0
            positions = [0]
            overlaps = [0]

            # there are more blocks, at least a second block:
            if batch_length < scene_instance_length:
                last_block = False
            # first block is the only block:
            else:
                last_block = True

            # until we have collected all blocks
            while (not last_block):
                # next block
                block_idx += 1

                # position candidate
                position = positions[block_idx - 1] + batch_length

                # for intermediate block overlapsuch that the first history_length-1 elements are shared with previous block
                # and the history_length's element is the first new frame (the overlapping part will be masked)
                overlap = history_length - 1

                # last block only:
                if position - overlap + batch_length >= scene_instance_length:
                    last_block = True
                    overlap = position + batch_length - scene_instance_length + 1
                    # ensure overlap is nonnegative and smaller than batch_length
                    assert overlap >= 0 and overlap < batch_length

                # update position to include overlap
                position -= overlap

                # ensure that beginning and end of block are within 0...scene_instance_length-1
                assert position >= 0 and position <= scene_instance_length - 1

                if last_block:
                    # ensure that last block really is last block
                    assert position == scene_instance_length - batch_length - 1

                positions.append(position)
                overlaps.append(overlap)

            # the length of both returned lists is the number of (batchlength long) blocks
            return positions, overlaps

        print('creating block positions and overlaps dict ')

        # initialize the dicitionary for filename->positions,overlap
        new_dict = {}

        # we need the scene instance lengths for each filename
        self.length_dict_()

        all_existing_files = glob.glob(self.pickle_path_pattern)
        all_existing_files = sorted(all_existing_files)
        for filename in tqdm(all_existing_files):
            positions, overlaps = calculate_block_positions_and_overlaps(self.length_dict[filename],
                                                                         self.params['batchlength'],
                                                                         self.params['historylength'])
            new_dict[filename] = (positions, overlaps)

        # load overall dictionary from file or, if file not existing, initialize it
        try:
            with open(pickle_path, 'rb') as handle:
                all_block_positions_and_overlaps_dict = pickle.load(handle)
        except IOError: #
            all_block_positions_and_overlaps_dict = {}

        # the to be pickled overall dictionary: dict(batchlength,historylength)->dict(filename->positions,overlaps)
        all_block_positions_and_overlaps_dict[(self.params['batchlength'], self.params['historylength'])] = new_dict

        with open(pickle_path, 'wb') as handle:
            pickle.dump(all_block_positions_and_overlaps_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)

    def fill_scene_instance_buffers(self):

        t_start_allfill = time()

        # if (len(self.scene_instance_buffers) == 0):
        #     complete_fill = True
        #     t_start_buffer_filling = time() # keep (only) this time measurement after profiling
        #     print('batchloader {} (epoch {}): filling empty buffer with {} scene instances...'.format(self.mode, self.epoch+1,
        #                                                                                self.params['sceneinstancebufsize']))
        # else:
        #     complete_fill = False

        runtime_filename = 0.
        runtime_instantiation = 0.
        runtime_appending = 0.
        runtime_lengthcalc = 0.
        # add more scene instance buffers until buffer full or all files consumed (e.g. in test mode for a single scene)
        while (len(self.scene_instance_buffers) < self.params['sceneinstancebufsize'] and len(self.filenames) > 0):

            t_start = time()
            # get next of the (in train mode shuffled) filenames
            filename = self.filenames.pop()
            runtime_filename += time()-t_start

            t_start = time()
            # create buffer and append it to buffers list
            scene_instance_buf = SceneInstanceBuffer(filename,  self, self.mode, self.params)
            runtime_instantiation += time()-t_start

            t_start = time()
            self.scene_instance_buffers.append(scene_instance_buf)
            if self.mode == 'train':
                self.scene_instance_buffers_last = np.append(self.scene_instance_buffers_last, 10000)
                self.scene_instance_buffers_remaining.append(len(scene_instance_buf))
            runtime_appending += time()-t_start

            t_start = time()
            # calculate the total number of blocks in alle scene instance buffers
            self.blocks_allbuffers += len(scene_instance_buf)
            runtime_lengthcalc += time()-t_start

        # if complete_fill:
        #     print('batchloader {} (epoch {}): ...filled buffer in {:.2f} sec'.format(self.mode, self.epoch+1,
        #                                                                   time()-t_start_buffer_filling))

        # runtime_fillbuffer = time()-t_start_allfill

        # print('batchloader: fill scene instance buffers took {:.2f} => filename {:.2f}, instantiation {:.2f}, appending {:.2f}, lengthcalc {:.2f}'
        #       .format(runtime_fillbuffer, runtime_filename, runtime_instantiation, runtime_appending, runtime_lengthcalc))

    def init_epoch(self, first=False):

        if first:
            self.epoch = 0
            self.scene_instance_buffers = []
            if self.mode == 'train':
                # data for biased sampling from scene instance buffer ids in training mode
                self.scene_instance_buffers_last = np.array([], dtype=np.int)
                self.scene_instance_buffers_remaining = []
        else:
            self.epoch += 1
            # ensure that the scene instance buffer list is empty
            assert not self.scene_instance_buffers

        # recover original list of all filenames
        self.filenames = self.filenames_all.copy()

        # set new seed [only applicable in training mode, valid and test scene instances are always processed in order]
        if self.mode == 'train':
            # set new seed (control self.seed for reproducible results)
            random.seed(self.seed * (self.epoch+1)) # to have new seed per epoch (also for validation set)

            # shuffle filenames (not required for valid/test sets)
            random.shuffle(self.filenames)

        # number of all blocks that can be fetched from all buffers
        self.blocks_allbuffers = 0

        self.batchid = -1 # will be increased by each __next__() call

        # the scene instance buffers should be empty, otherwise we should not init an epoch
        assert len(self.scene_instance_buffers) == 0

        # print('batchloader (mode {}) is prepared for epoch {}'.format(self.mode, self.epoch+1))

    def __iter__(self):
        return self

    def __next__(self):

        t_start_nextbatch = time()

        t_start = time()
        # all batches created => next epoch or stop iteration
        if self.batchid > 0 and len(self.scene_instance_buffers) == 0:
            assert self.batches_per_epoch == self.batchid + 1 # TODO: remove if multiproc
            # start next epoch:
            if self.epoch + 1 < self.params['maxepochs']:
                self.init_epoch() # also sets self.batchid = -1
            # but only until maxepochs is reached
            else:
                raise StopIteration

        # fills buffers and update self.blocks_allbuffers
        # (only effective if a scene instance buffer is empty or last filename consumed)
        # refilling is reuiqred not only during epoch init but also before each batch construction
        self.fill_scene_instance_buffers() # randomness enters here via shuffled filenames

        # create a batch (potentially smaller than params['batchsize'] if it is the last one
        self.batchid += 1
        blockid = 0

        if self.mode == 'train':
            self.scene_instance_buffers_last += 1

        # free memory (unreferenced scene instance buffers) after every few batches
        if self.batchid % self.free_memory_batches == 0:
            gc.collect()

        runtime_sample_index = 0.
        runtime_next_block = 0.
        runtime_postproc_block = 0.
        runtime_postproc_block_assignment = 0.
        runtime_postproc_block_listops = 0.
        runtime_remove_block = 0.

        runtime_start = time()-t_start

        # add blocks to our batch until we have filled the batch or there are no blocks in the buffer left (last batch)
        while (blockid < self.params['batchsize'] and len(self.scene_instance_buffers) > 0):

            t_start = time()
            # training mode: get random scene instance buffer:
            if self.mode == 'train':
                    probs = self._scene_instance_buffer_probs()
                    sibuf_id = np.random.choice(len(self.scene_instance_buffers), size=1, replace=False, p=probs)[0]
                    # old version (without respecting batch correlation, without biasing towards longer sequences)
                    # sibuf_id = random.randint(0, len(self.scene_instance_buffers)-1)

            # validation/test mode: simply take the first scene instance:
            else:
                sibuf_id = 0

            runtime_sample_index += time()-t_start

            # extract next block from chosen scene instance (if existing)
            t_start = time()
            scene_instance_buffer = self.scene_instance_buffers[sibuf_id]
            buffer_output = next(scene_instance_buffer.iter, None)
            runtime_next_block += time()-t_start

            t_start = time()
            remove_buffer = False
            if buffer_output is not None:
                t_start_inner = time()
                self.batch_x[blockid, :, :], self.batch_y[blockid, :, :, :] = buffer_output
                runtime_postproc_block_assignment += time()-t_start_inner

                t_start_inner = time()
                # we fetched a valid block => update data for biased sampling
                if self.mode == 'train':
                    self.scene_instance_buffers_last[sibuf_id] = 0
                    self.scene_instance_buffers_remaining[sibuf_id] -= 1
                    if self.scene_instance_buffers_remaining[sibuf_id] == 0:
                        remove_buffer = True
                    else:
                        remove_buffer = False
                blockid += 1
                runtime_postproc_block_listops = time()-t_start_inner
            else:
                remove_buffer = True

            runtime_postproc_block += time()-t_start

            t_start = time()
            # remove scene instance that is completely used
            if remove_buffer:
                self.scene_instance_buffers.pop(sibuf_id)
                if self.mode == 'train':
                    self.scene_instance_buffers_last = np.delete(self.scene_instance_buffers_last, sibuf_id)
                    self.scene_instance_buffers_remaining.pop(sibuf_id)

            runtime_remove_block += time()-t_start

        # handling a case (relevant e.g. for blocklength 2000) that has a single (empty) remaining scene instance
        if blockid==0: # and remove_buffer
            raise StopIteration

        t_start = time()
        effective_batchsize = blockid # after the while loop blockid corresponds to the no of blocks taken

        # update number of remaining blocks
        self.blocks_allbuffers -= effective_batchsize

        # determine effective batch (size batchsize or for last batch smaller)
        if effective_batchsize < self.params['batchsize']:
            effective_batch_x = self.batch_x[:effective_batchsize, :, :]
            effective_batch_y = self.batch_y[:effective_batchsize, :, :, :]
        else:
            effective_batch_x = self.batch_x
            effective_batch_y = self.batch_y

        runtime_finish = time()-t_start

        runtime_nextbatch_total = time()-t_start_nextbatch
        # print('batchloader: total time to get the batch was {:.2f} => start {:.2f}, sample {:.2f}, next block {:.2f}, postproc block {:.2f} (assignment {:.2f}, listops {:.2f}), remove block {:.2f}, finish {:.2f}'
        #       .format(runtime_nextbatch_total, runtime_start, runtime_sample_index, runtime_next_block, runtime_postproc_block, runtime_postproc_block_assignment, runtime_postproc_block_listops, runtime_remove_block, runtime_finish))

        # we need to return a copied version of the batch in order not to overwrite it when creating the next batch
        return np.copy(self._input_standardization_if_wanted(effective_batch_x)), np.copy(effective_batch_y)

    # calculate probabilities that weight each scene instance buffer with its sequence lengt and also decrease weights of
    # scene instances that were sampled in the previous batch, i.e., decrease ensure that the last batches do not consist
    # of several batches of the last remaining long sequences, and also generally correlation between any two directly
    # successive batches are removed
    # optional improvement: decrease probability also if a block was taken two batches before (e.g. weight by inverse last seen)
    def _scene_instance_buffer_probs(self):
        assert len(self.scene_instance_buffers_remaining) == len(self.scene_instance_buffers_last) == len(self.scene_instance_buffers)
        remaining_squared = np.array(self.scene_instance_buffers_remaining)**2
        previous = self.scene_instance_buffers_last <= 1
        if previous.any():
            # remove probability for previously taken scene instance
            remaining_squared_reduced_previous = remaining_squared.copy().astype(np.float)
            remaining_squared_reduced_previous[previous] *= 0.001
        else:
            remaining_squared_reduced_previous = remaining_squared
        probs = remaining_squared_reduced_previous / remaining_squared_reduced_previous.sum()
        return probs

# testing:
if __name__ == '__main__':

    # ### train
    # mode = 'train'
    # folds = [1, 2, 3, 4, 5, 6] # final model
    # folds = [1, 2, 4, 5, 6] # lvl1
    # folds = [1, 2, 3, 5, 6] # lvl2
    # folds = [1, 3, 4, 5, 6] # lvl3
    # scenes = list(range(1, 80 + 1))  # -1 # all scenes (80 for train)

    # ### test
    mode = 'test'
    folds = [7, 8] # test folds
    scenes = list(range(1, 168 + 1)) # all 168 test scenes

    # ### val/toy
    # mode = 'val'
    # folds = [3]
    # folds = [4]
    # folds = [2]
    # scenes = [1]
    # scenes = list(range(1, 80 + 1))  # -1 # all scenes (80 for train)

    inputstd = True
    # inputstd = False
    params = {'sceneinstancebufsize': 200, #2000,
              'historylength': 1025,
              'batchsize': 128,
              'batchlength': 2000, #2500,
              'instantlabels': False,
              'maxepochs': 1,
              'noinputstandardization': not inputstd,
              'mask_value': -1}

    print('batchloader ({} mode, inputstd={}, folds {}, scenes {}) before initialization.....'.format(mode, inputstd,
                                                                                folds, scenes))

    t_start_allbatches = time()
    batchloader = BatchLoader(params=params, mode=mode, fold_nbs=folds,
                              scene_nbs=scenes, batchsize=params['batchsize'],
                              seed=1)  # seed for testing

    # for checking input standardization
    no_batches = batchloader.batches_per_epoch * params['maxepochs']
    mean_vec_batches = np.zeros((160, no_batches))
    std_vec_batches = np.zeros_like(mean_vec_batches)
    no_unmasked_features = np.zeros(no_batches, dtype=np.int)

    print('iterating once over the data set:')
    t_start_allbatches = time()
    batchcount = 0
    t_start_batch = time()

    for batch_x, batch_y in batchloader:
        batchcount += 1
        print('received batch (size {}) {}/{} in epoch {}/{} in {:.2f} sec'.format(batch_x.shape[0], batchloader.batchid % batchloader.batches_per_epoch + 1,
                                                                                batchloader.batches_per_epoch, batchloader.epoch + 1, params['maxepochs'],
                                                                                time() - t_start_batch))
        batch_x_unmasked = batch_x[batch_y[:, :, 0, 1] != params['mask_value']]
        no_unmasked_features[batchcount-1] = len(batch_x_unmasked)
        print('batch has {} unmasked of {} total feature vectors'.
              format(no_unmasked_features[batchcount-1],
                     batch_x.shape[0]*batch_x.shape[1]))
        mean_vec_batches[:, batchcount-1] = np.mean(batch_x_unmasked, axis=0)
        std_vec_batches[:, batchcount-1] = np.std(batch_x_unmasked, axis=0)
        t_start_batch = time()
    assert batchcount == batchloader.batches_per_epoch * params['maxepochs']
    print('...done receiving {} batches of {} epochs in {:.2f} seconds'.format(batchcount, params['maxepochs'],
                                                                               time() - t_start_allbatches))

    print('calculating mean and std of whole data set to validate input standardization:')
    # # assumption of the commented out code: size and length of all batches is equal (pot. smaller last batch is negligible)
    # mean_vec_total = np.mean(mean_vec_batches, axis=1)
    # var_vec_total = 1./batchcount * np.sum((std_vec_batches**2 + (mean_vec_batches-mean_vec_total[:, np.newaxis])**2), axis=1)
    # corrected  code (compared to commented out above): taking into account masking of features
    mean_vec_total = 1./np.sum(no_unmasked_features) * np.sum(no_unmasked_features[np.newaxis, :] * mean_vec_batches,
                                                              axis=1)
    var_vec_total = 1. / np.sum(no_unmasked_features) * np.sum(no_unmasked_features[np.newaxis, :] *
                                                               (std_vec_batches ** 2 +
                                                                (mean_vec_batches - mean_vec_total[:, np.newaxis]) ** 2),
                                                               axis=1)
    std_vec_total = np.sqrt(var_vec_total)
    print('total mean vector (should be close to component-wise zero): \n{}'.format(mean_vec_total))
    print('total std vector (should be close to component-wise unity): \n{}'.format(std_vec_total))


"""
Summation of primes

Problem 10
The sum of the primes below 10 is 2 + 3 + 5 + 7 = 17.

Find the sum of all the primes below two million.
"""
import time
import numpy as np

def sieveOfEratosthenes(n=11):
    """
    The sieve of Eratosthenes: find all the prime numbers less than or equal to n

    :param n: an integer n > 1
    :return: all prime numbers from 2 through n.
    """
    sieve = [True for x in range(n + 1)]  # prime number mask
    sieve[0] = sieve[1] = False  # 0 and 1 are not prime numbers

    i = 2
    while i ** 2 <= n:
        if sieve[i]:
            for j in range(i ** 2, n + 1, i):
                sieve[j] = False
        i += 1

    return [i for i in range(n + 1) if sieve[i]]

def summationOfPrimesLessThan(limit=2_000_000):
    return sum(sieveOfEratosthenes(limit))

if __name__ == '__main__':
    print(summationOfPrimesLessThan())


import numpy as np; import pandas as pd
import json
pd.options.display.max_rows = 10

class practice:

    def movielens():
        unames = ['user_id', 'gender', 'age', 'occupation', 'zip']
        users = pd.read_table('C:\\Users\\kjsj9\\Desktop\\pydata-book-2nd-edition\\pydata-book-2nd-edition\\datasets\\movielens\\users.dat',
                              sep = '::',
                            header = None, names = unames)

        rnames = ['user_id', 'movie_id', 'rating', 'timestamp']
        ratings = pd.read_table('C:\\Users\\kjsj9\\Desktop\\pydata-book-2nd-edition\\pydata-book-2nd-edition\\datasets\\movielens\\ratings.dat',
                                sep = '::',
                            header = None, names = rnames)

        mnames = ['move_id', 'title', 'genres']
        movies = pd.read_table('C:\\Users\\kjsj9\\Desktop\\pydata-book-2nd-edition\\pydata-book-2nd-edition\\datasets\\movielens\\movies.dat',
                            sep = '::',
                            header = None, names = mnames)

        #print("\nfive users:\n", users[:5])
        #print("\nfive ratings:\n", ratings[:5])
        #print("\nfive movies:\n", movies[:5])

        print("\nratings:\n", ratings)
        print(movies)
        print(users)

        data = pd.merge(pd.merge(ratings, users, how = 'outer'), movies, how = 'outer')
        print("\nmerged data:\n", data)

        print("\nfirst user:\n", data.iloc[0])

        mean_ratings = data.pivot_table('rating', index = 'title',
                                        columns = 'gender', aggfunc = 'mean')
        print("\nmean ratings by gender of movies:\n", mean_ratings[:6])

        ratings_by_title = data.groupby('title').size()
        print(ratings_by_title[:10])

        active_titles = ratings_by_title.index[ratings_by_title >= 250]
        print("\nactive titles:\n", active_titles)

        top_f_ratings = mean_ratings.sort_values(by = 'F', ascending = False)
        print("\n top 10 female top ratings:\n", top_f_ratings[:10])

        mean_ratings['diff'] = mean_ratings['M'] - mean_ratings['F']

        sorted_by_diff = mean_ratings.sort_values(by = 'diff')

        print("\npreferred by female to male:\n", sorted_by_diff)

    def vote():
        fec = pd.read_csv('C:\\Users\\kjsj9\\Desktop\\pydata-book-2nd-edition\\pydata-book-2nd-edition\\datasets\\fec\\P00000001-ALL.csv')
        print("\nfec info:\n", fec.info())

        print("\n123456th fec:\n", fec.iloc[123456])

        unique_cands = fec.cand_nm.unique()

        parties = {'Bachmann, Michelle' : 'Republican',
                 'Cain, Herman' : 'Republican',
                 'Gingrich, Newt' : 'Republican',
                 'Huntsman, Jon' : 'Republican',
                 'Johnson, Gary Earl' : 'Republican',
                 'McCotter, Thaddeus G' : 'Republican',
                 'Obama, Barack' : 'Democrat',
                 'Paul, Ron' : 'Republican',
                 'Pawlenty, Timothy' : 'Republican',
                 'Perry, Rick' : 'Republican',
                 "Roemer, Charles E. 'Buddy' III" : 'Republican',
                 'Romney, Mitt' : 'Republican',
                 'Santorum, Rick' : 'Republican'}

        print("\ncandidate 123456 to 123461:\n", fec.cand_nm[123456:123461])

        fec['party'] = fec.cand_nm.map(parties)

        print("\nparty counts:\n", fec['party'].value_counts())

        print(("give or return:\n", (fec.contb_receipt_amt > 0).value_counts()))

        fec = fec[fec.contb_receipt_amt > 0]

        fec_mrbo = fec[fec.cand_nm.isin(['Obama, Barack', 'Romney, Mitt'])]

        print("\n 10 occupations: \n", fec.contbr_occupation.value_counts()[:10])

        by_occupation = fec.pivot_table('contb_receipt_amt',
                                        index ='contbr_occupation',
                                        columns = 'party', aggfunc = 'sum')
        over_2mm = by_occupation[by_occupation.sum(1) > 2000000]

        print("\nover 2mil:\n", over_2mm)

        print(over_2mm.plot(kind = 'barh'))

        def get_top_amounts(group, key, n = 5):
            totals = group.groupby(key)['contb_receipt_amt'].sum()
            return totals.nlargest(n)

        grouped = fec_mrbo.groupby('cand_nm')

        print(grouped.apply(get_top_amounts, 'contbr_occupation', n =7))

        grouped.apply(get_top_amounts, 'contbr_employer', n = 10)

        #기부금액
        bins = np.array([0, 1, 10, 100, 1000, 10000,
                         100000, 1000000, 10000000])
        labels = pd.cut(fec_mrbo.contb_receipt_amt, bins)

        print("\ncateogires by donation size:\n", labels)

        grouped = fec_mrbo.groupby(['cand_nm', labels])
        print("\nunstacked:\n", grouped.size().unstack(0))

        #weekly 기부금액
        grouped = fec_mrbo.groupby(['cand_nm', 'contbr_st'])
        totals = grouped.contb_receipt_amt.sum().unstack(0).fillna(0)
        totals = totals[totals.sum(1) > 100000]

        print("\nweekly:\n", totals[:10])

        percent = totals.div(totals.sum(1), axis =0)
        print("\npercentage:\n", percent[:10])

    if __name__ == "__main__":
        #movielens()
        vote()


# Advent of Code 2021
# --- Day 1: Sonar Sweep ---

# For Part 1

sonars = []
with open('Advent/day_01_input.txt') as sonar_file:
    sonars = sonar_file.readlines()

def sonar_increaser(waves):
    depth_increase_counter = 0
    for index in range(0, len(waves)):
        if int(waves[index - 1]) < int(waves[index]):
            depth_increase_counter += 1
            # print(f"{waves[index - 1]} < {waves[index]}", end="")
    print(f"Final answer is: {depth_increase_counter}")

sonar_increaser(sonars)

# For Part 2
def triple_measurements(waves):
    triple_waves = []
    # depth_increase_counter = 1
    # index = 1
    counter = -1
    for index in range(0, len(waves) - 2):
        # letter = waves[index] + waves[index + 1] + waves[index + 2]
        letter = int(waves[index]) + int(waves[index + 1]) + int(waves[index + 2])
        triple_waves.append(letter)
        counter += 1
    print(f"Triple wave counter: {counter}")
    return triple_waves

reading = triple_measurements(sonars)
sonar_increaser(reading)


from flask import Flask, render_template, request, Response, send_from_directory, redirect, url_for, jsonify
from flask_sslify import SSLify
from flask_cors import CORS
import os
from PIL import Image
import json
import base64
import cv2
import numpy as np
#from src import classifier
import web_face_recognition
import time
import subprocess as sp
import requests
import threading
from pymongo import MongoClient
from threading import Thread

app = Flask(__name__)
sslify = SSLify(app)
CORS(app)
save_path = str('/work/MachineLearning/my_dataset/wos_event/')
client = MongoClient('mongodb://localhost:27017/')
#Accessing databases
db = client['attendance']
name_list = db['name_list']

# added to put object in JSON
class Object(object):
    def __init__(self):
        self.name = "Queue list"

    def toJSON(self):
        return json.dumps(self.__dict__)

@app.after_request
def after_request(response):
    response.headers.add('Access-Control-Allow-Origin', '*')
    response.headers.add('Access-Control-Allow-Headers', 'Content-Type,Authorization')
    response.headers.add('Access-Control-Allow-Methods', 'GET,POST,OPTIONS')
    return response

@app.route('/')
def index():
    return Response(os.getcwd())

@app.route('/recognition')
def start():
    return Response(open('/work/MachineLearning/facenet_m360/contributed/templates/single_shot_recognition.html').read(), mimetype="text/html")

@app.route('/enrol', methods=['POST'])
def enrol():
    try:
        if request.method == 'POST':
            print('POST /enrol success!')

        image_file = json.loads(request.data)
        name = str(image_file['id'])

        if not os.path.exists(os.path.join(save_path+name)):
            os.mkdir(save_path+name)

        web_face_recognition.append_queue(name)
        count = 0
        for images in image_file['data']:
            filename = name + str(count)
            img = base64.b64decode(images)
            img_array = np.fromstring(img, np.uint8)
            imgdata = cv2.imdecode(img_array, cv2.IMREAD_COLOR)
            rgb_frame = imgdata[:, :, ::-1]
            img = Image.fromarray(rgb_frame, "RGB")
            if img is not None:
                img.save(os.path.join(save_path+name+'/'+filename+".jpg"))
            count += 1

        if web_face_recognition.run() is False:
            web_face_recognition.enrol(incremental=True)

        return redirect(url_for('getStatus', _external=True, _scheme='https'))

    except Exception as e:
        print('POST /enrol error : %s' % e)
        return e

@app.route('/getStatus', methods=['GET', 'POST'])
def getStatus():
    try:
        output = []
        queue = web_face_recognition.get_status()

        return queue

    except Exception as e:
        print('Check status failed : %s' % e)
        return e

@app.route('/recognition_result', methods=['POST'])
def face_recognition():
    try:
        image_file = json.loads(request.data.decode('utf-8'))
        img = base64.b64decode(image_file['data'])
        img_array = np.fromstring(img, np.uint8)
        imgdata = cv2.imdecode(img_array, cv2.IMREAD_COLOR)
        boxes = web_face_recognition.recognize(imgdata)
        boxes['name'] = name_list.find_one({'person': boxes['name']})

        return boxes

    except Exception as e:
        print('Recognition failed : %s' % e)
        return e

if __name__ == '__main__':
    #start_runner()
    app.run(host="0.0.0.0", port=8081, threaded=True, ssl_context='adhoc')
    #app.run(host='0.0.0.0', port=8081, threaded=True, debug=True, ssl_context="adhoc")


from typing import Callable

import torch
from torch import Tensor

from thre3d_atom.rendering.volumetric.render_interface import (
    Rays,
    ProcessedPointsOnRays,
    RenderOut,
)
from thre3d_atom.utils.constants import (
    ZERO_PLUS,
    INFINITY,
    EXTRA_ACCUMULATED_WEIGHTS,
    EXTRA_POINT_WEIGHTS,
    EXTRA_POINT_DEPTHS,
    EXTRA_POINT_OCCUPANCIES,
    EXTRA_POINT_DENSITIES,
    EXTRA_SAMPLE_INTERVALS,
    EXTRA_DISPARITY,
)

def density2occupancy_pb(densities: Tensor, deltas: Tensor) -> Tensor:
    """computes occupancy values from density values in the range [0, inf).
    The occupancy (being a probability) is always strictly between [0, 1].
    This function is physically based, and can be derived from Lambert's law"""
    return 1.0 - torch.exp(-(densities * deltas))

def accumulate_radiance_density_on_rays(
    processed_points: ProcessedPointsOnRays,
    rays: Rays,
    stochastic_density_noise_std: float = 1.0,
    density2occupancy: Callable[[Tensor, Tensor], Tensor] = density2occupancy_pb,
    radiance_hdr_tone_map: Callable[[Tensor], Tensor] = torch.sigmoid,
    white_bkgd: bool = True,
    extra_debug_info: bool = False,
) -> RenderOut:
    dtype, device = processed_points.points.dtype, processed_points.points.device

    # unpack the radiance and density from the processed points
    raw_radiance, raw_density = (
        processed_points.points[..., :-1],
        processed_points.points[..., -1],
    )

    # compute sample-intervals for ray-time integration
    deltas = processed_points.depths[..., 1:] - processed_points.depths[..., :-1]
    inf_delta = torch.full(
        size=(*deltas.shape[:-1], 1), fill_value=INFINITY, dtype=dtype, device=device
    )
    deltas = torch.cat([deltas, inf_delta], dim=-1)  # [N_rays, N_samples]
    # need proper deltas in coordinate domain space, hence normalizing ray-dir norms
    deltas = deltas * rays.directions[..., None, :].norm(dim=-1)

    # add stochastic noise to density if requested :).
    # can be used to regularize repr during training (kind-of-sort-of prevents floater artifacts).
    density_noise = (
        torch.randn(raw_density.shape, dtype=dtype, device=device)
        * stochastic_density_noise_std
    )
    alpha = density2occupancy(raw_density + density_noise, deltas)

    # compute the radiance weights for accumulation along the ray
    ones = torch.ones((alpha.shape[0], 1), dtype=dtype, device=device)
    weights = alpha * torch.cumprod(torch.cat([ones, 1.0 - alpha], -1), -1)[:, :-1]

    # accumulate the predicted radiance values of the samples using the computed alphas
    colour = radiance_hdr_tone_map(raw_radiance)
    # dims below: [N, NUM_COLOUR_CHANNELS]
    colour_render = torch.sum(colour * weights[..., None], dim=-2)

    # sum of weights along each ray, should ideally be 1.0 everywhere
    acc_render = torch.sum(weights, dim=-1, keepdim=True)

    if white_bkgd:
        # add a white background if requested. Mathematically, note that we assume
        # the background to be emitting a solid bright white colour RGB=(1.0, 1.0, 1.0)
        # hence the simplification of the alpha-composition formula :)
        colour_render = colour_render + (1 - acc_render)

    # compute depth_render and disparity_render (inverse depth)
    depth_render = (processed_points.depths * weights).sum(dim=-1, keepdims=True)
    disparity_render = 1.0 / torch.maximum(
        torch.full(acc_render.shape, ZERO_PLUS, device=depth_render.device),
        depth_render / acc_render,
    )

    # additional renders
    extra_dict = {
        EXTRA_DISPARITY: disparity_render,
        EXTRA_ACCUMULATED_WEIGHTS: acc_render,
    }

    if extra_debug_info:
        # adds very big tensor buffers
        # num samples per ray (or ray-chunk). Please only use for debugging purposes
        extra_dict.update(
            {
                EXTRA_POINT_DENSITIES: raw_density,
                EXTRA_POINT_OCCUPANCIES: alpha,
                EXTRA_POINT_WEIGHTS: weights,
                EXTRA_POINT_DEPTHS: processed_points.depths,
                EXTRA_SAMPLE_INTERVALS: deltas,
            }
        )

    return RenderOut(
        colour=colour_render,
        depth=depth_render,
        extra=extra_dict,
    )