"""
REM records classes and functions.
"""

from bs4 import BeautifulSoup
import datetime
import os
import sys
from random import randint
import re

import PySimpleGUI as sg
import dateutil
import numpy as np
import pandas as pd
import pdfkit
from jinja2 import Environment, FileSystemLoader

import REM.constants as mod_const
import REM.data_manipulation as mod_dm
import REM.database as mod_db
import REM.elements as mod_elem
import REM.layouts as mod_lo
import REM.secondary as mod_win2
from REM.client import logger, server_conn, settings, user

class RecordGroup:
    """
    Attributes:
        name (str): name of the record group.

        program_record (bool): records in the record group are program records [Default: True].

        permissions (dict): dictionary mapping permission rules to permission groups
    """

    def __init__(self, name, entry):
        """
        Record group configuration.

        Arguments:
            name (str): name of the record group.

            entry (dict): configuration entry for the record group.
        """
        self.name = name

        # Specify whether a record group is a program record or an external record
        try:
            self.program_record = bool(int(entry['ProgramRecord']))
        except KeyError:  # parameter not specified
            self.program_record = True
        except ValueError:  # wrong data type provided to parameter
            msg = 'Configuration Error: "ProgramRecord" must be either 0 (False) or 1 (True)'
            logger.error('RecordGroup {NAME}: {MSG}'.format(NAME=self.name, MSG=msg))
            mod_win2.popup_error(msg)

            self.program_record = True

        # Set record menu parameters
        try:
            menu = entry['Menu']
        except KeyError:
            menu = {'MenuTitle': self.name, 'MenuGroup': None}
            self.show_menu = False
        else:
            self.show_menu = True
            if 'MenuGroup' not in menu:
                menu['MenuGroup'] = None
            if 'MenuTitle' not in menu:
                menu['MenuTitle'] = self.name

        self.menu_title = menu['MenuTitle']
        self.menu_group = menu['MenuGroup']

        try:
            permissions = entry['Permissions']
        except KeyError:
            self.permissions = {'view': None, 'create': None, 'edit': None, 'delete': None, 'upload': None}
        else:
            self.permissions = {'view': permissions.get('View', None),
                                'create': permissions.get('Create', None),
                                'edit': permissions.get('Edit', None),
                                'delete': permissions.get('Delete', None),
                                'upload': permissions.get('Upload', None),
                                }

        try:
            self.id_code = entry['IDCode']
        except KeyError:
            mod_win2.popup_error('RecordGroup {NAME}: configuration missing required parameter "IDCode"'
                                 .format(NAME=name))
            sys.exit(1)

        # Database import rules
        try:
            import_rules = entry['ImportRules']
        except KeyError:
            mod_win2.popup_error('RecordGroup {NAME}: configuration missing required parameter "ImportRules"'
                                 .format(NAME=name))
            sys.exit(1)
        else:
            for import_table in import_rules:
                import_rule = import_rules[import_table]

                if 'Columns' not in import_rule:
                    mod_win2.popup_error('RecordsEntry {NAME}: configuration missing required "ImportRules" {TBL} '
                                         'parameter "Columns"'.format(NAME=name, TBL=import_table))
                    sys.exit(1)
                if 'Filters' not in import_rule:
                    import_rule['Filters'] = None

        self.import_rules = import_rules

        # Database export rules
        if self.program_record:
            try:
                export_rules = entry['ExportRules']
            except KeyError:
                self.export_rules = {}
                for table in self.import_rules:
                    table_entry = self.import_rules[table]
                    import_columns = table_entry['Columns']

                    export_columns = {}
                    for import_column in import_columns:
                        import_alias = import_columns[import_column]
                        if isinstance(import_alias, list):
                            for export_column in import_alias:
                                export_columns[export_column] = import_column
                        else:
                            export_columns[import_alias] = import_column

                    self.export_rules[table] = {'Columns': export_columns}
            else:
                self.export_rules = {}
                for export_table in export_rules:
                    export_rule = export_rules[export_table]

                    if 'Columns' not in export_rule:
                        mod_win2.popup_error('RecordsEntry {NAME}: configuration missing required "ExportRules" {TBL} '
                                             'parameter "Columns"'.format(NAME=name, TBL=export_table))
                        sys.exit(1)

                    self.export_rules[export_table] = export_rule
        else:  # only program records can export records to the database
            self.export_rules = {}

        # Record association rules
        try:
            association_rules = entry['AssociationRules']
        except KeyError:
            logger.info('RecordGroup {NAME}: no association rules specified for the record group'
                        .format(NAME=self.name))
            association_rules = {}

        self.association_rules = {}
        for rule_name in association_rules:
            rule = association_rules[rule_name]

            if 'Primary' not in rule:
                msg = 'RecordGroup {NAME}: AssociationRule {RULE} is missing required parameter "Primary"' \
                    .format(NAME=self.name, RULE=rule_name)

                raise AttributeError(msg)

            if 'ReferenceTable' not in rule:
                msg = 'RecordGroup {NAME}: AssociationRule {RULE} is missing required parameter "ReferenceTable"' \
                    .format(NAME=self.name, RULE=rule_name)

                raise AttributeError(msg)

            if 'Title' not in rule:
                rule['Title'] = rule_name

            if 'AssociationType' in rule:
                assoc_type = rule['AssociationType']
                if assoc_type not in ('parent', 'child', 'reference'):
                    msg = 'RecordGroup {NAME}: unknown association type {TYPE} provided to association rule {RULE}' \
                        .format(NAME=self.name, TYPE=assoc_type, RULE=rule_name)

                    raise AttributeError(msg)

            self.association_rules[rule_name] = {'AssociationType': rule.get('AssociationType', 'reference'),
                                                 'Primary': rule.get('Primary'),
                                                 'ReferenceTable': rule.get('ReferenceTable'),
                                                 'HardLink': rule.get('HardLink', None)}

        # Import table layout configuration
        try:
            self.import_table = entry['ImportTable']
        except KeyError:
            msg = 'RecordGroup {NAME}: missing configuration parameter "ImportTable"'.format(NAME=self.name)
            logger.warning(msg)

            self.import_table = {}

        # Record layout configuration
        try:
            self.record_layout = entry['RecordLayout']
        except KeyError:
            mod_win2.popup_error('RecordsEntry {NAME}: configuration missing required parameter "RecordLayout"'
                                 .format(NAME=name))
            sys.exit(1)

        try:
            self.id_column = entry['IDColumn']
        except KeyError:
            self.id_column = 'RecordID'

        try:
            self.date_column = entry['DateColumn']
        except KeyError:
            self.date_column = 'RecordDate'

        try:
            self.deleted_column = entry['DeletedColumn']
        except KeyError:
            self.deleted_column = 'IsDeleted'

    def _format_filter_set(self, filter_rules):
        """
        Format a set of database filters.
        """
        operators = {'=', '!=', '>', '<', '>=', '<=', 'IN', 'NOT IN'}

        filters = []
        for filter_column in filter_rules:
            filter_entry = filter_rules[filter_column]

            import_column = self.map_column(filter_column)

            try:
                operator = filter_entry[0].upper()
            except (IndexError, AttributeError):
                msg = 'no valid operator set for import filter {COL}'.format(COL=filter_column)
                logger.error('RecordGroup {NAME}: {MSG}'.format(NAME=self.name, MSG=msg))

                continue
            else:
                if operator not in operators:
                    msg = 'no valid operator set for import filter {COL}'.format(COL=filter_column)
                    logger.error('RecordGroup {NAME}: {MSG}'.format(NAME=self.name, MSG=msg))

                    continue

            try:
                parameters = filter_entry[1:]
            except IndexError:
                msg = 'one or more import values required for filter column {COL}'.format(COL=filter_column)
                logger.error('RecordGroup {NAME}: {MSG}'.format(NAME=self.name, MSG=msg))

                continue
            else:
                if len(parameters) == 1:
                    parameters = parameters[0]
                else:
                    parameters = tuple(parameters)

            if isinstance(parameters, list) or isinstance(parameters, tuple):
                values = ['?' for _ in parameters]
                value = '({VALS})'.format(VALS=', '.join(values))
            else:
                value = '?'

            if operator in ('IN', 'NOT IN') and 'NULL' not in parameters:
                filters.append(('({COL} {OPER} {VAL} OR {COL} IS NULL)'
                                .format(COL=import_column, OPER=operator, VAL=value), parameters))
            else:
                filters.append(('{COL} {OPER} {VAL}'
                                .format(COL=import_column, OPER=operator, VAL=value), parameters))

        return filters

    def _format_import_columns(self, import_rules: dict = None):
        """
        Format columns for querying.
        """
        converter_funcs = {'CAST', 'CONVERT'}
        merge_funcs = {'COALESCE', 'CONCAT'}

        import_rules = self.import_rules if import_rules is None else import_rules
        database = settings.prog_db if self.program_record else settings.dbname

        columns = []
        for import_table in import_rules:
            import_rule = import_rules[import_table]
            table_columns = user.table_schema(database, import_table)

            try:
                modifiers = import_rule['Modifiers']
            except KeyError:
                modifiers = {}

            import_columns = import_rule['Columns']
            for import_column in import_columns:
                column_alias = import_columns[import_column]

                col_mod = modifiers.get(import_column, None)
                if col_mod:
                    sql_func = col_mod['Function'].upper()
                    sql_args = col_mod['Arguments']

                    if sql_func in merge_funcs:
                        merge_args = []
                        for comp in sql_args:
                            if comp in table_columns:
                                merge_arg = '{TBL}.{COL}'.format(TBL=import_table, COL=comp)
                            else:
                                merge_arg = "'{}'".format(comp)

                            merge_args.append(merge_arg)

                        if sql_func == 'CONCAT':  # CONCAT is only supported in SQL Server >= 2012
                            oper = '({ARGS})'.format(ARGS='+'.join(merge_args))
                        else:
                            oper = '{FUNC}({ARGS})'.format(FUNC=sql_func, ARGS=','.join(merge_args))
                    elif sql_func in converter_funcs:
                        try:
                            dtype, col_size = sql_args
                        except ValueError:
                            try:
                                dtype = sql_args
                            except KeyError:
                                logger.warning('unable to modify column {COL} from import table {TBL} - datatype is a '
                                               'required argument for {FUNC} type modifiers'
                                               .format(COL=import_column, TBL=import_table, FUNC=sql_func))
                                dtype = col_size = None
                            else:
                                col_size = None

                        if dtype is not None:
                            if col_size is not None:
                                oper = '{FUNC}({TBL}.{COL} AS {DTYPE}({SIZE}))' \
                                        .format(FUNC=sql_func, TBL=import_table, COL=import_column, DTYPE=dtype, SIZE=col_size)
                            else:
                                oper = '{FUNC}({TBL}.{COL} AS {DTYPE})'.format(FUNC=sql_func, TBL=import_table, COL=import_column, DTYPE=dtype)
                        else:
                            oper = '{TBL}.{COL}'.format(TBL=import_table, COL=import_column)
                    else:
                        oper = '{TBL}.{COL}'.format(TBL=import_table, COL=import_column)
                else:
                    oper = '{TBL}.{COL}'.format(TBL=import_table, COL=import_column)

                column = '{OPER} AS {ALIAS}'.format(OPER=oper, ALIAS=column_alias)
                columns.append(column)

        return columns

    def _set_database(self, database: str = None):
        """
        Set the import database.
        """
        program_db = settings.prog_db
        if self.program_record:
            db = program_db
        else:
            if database:
                if database == program_db:
                    db = program_db
                elif database in settings.alt_dbs:
                    db = database
                else:
                    msg = 'database {DB} is not an available program database'.format(DB=database)
                    logger.warning('RecordGroup {NAME}: {MSG}'.format(NAME=self.name, MSG=msg))

                    db = settings.dbname
            else:
                db = settings.dbname

        return db

    def map_column(self, column):
        """
        Convert from record column to database column.
        """
        import_rules = self.import_rules
        db_col = mod_db.get_import_column(import_rules, column)

        return db_col

    def unique_values(self, field, sort: bool = False, database: str = None):
        """
        Extract the unique values of a given record field from the database.

        Arguments:
            field (str): name of the record field to retrieve values for.

            sort (bool): sort results [default: False].

            database (str): load unique values from the provided database.
        """
        import_rules = self.import_rules
        db = self._set_database(database)

        tables = mod_db.format_tables(import_rules)
        filters = mod_db.format_import_filters(import_rules)
        db_col = mod_db.get_import_column(import_rules, field)
        if sort:
            order_by = db_col
        else:
            order_by = None

        # Find the date of the most recent transaction prior to current date
        query = mod_db.prepare_sql_query(tables, columns=db_col, filter_rules=filters, order=order_by, distinct=True)
        unique_df = user.read_db(*query, database=db)
        unique_values = unique_df.iloc[:, 0].tolist()

        return unique_values

    def load_records(self, id_list, use_import_rules: bool = True, filters: dict = None, import_rules: dict = None,
                     database: str = None):
        """
        Load database records from a list of provided record IDs.

        Arguments:
            id_list (list): load records with these record IDs from the database.

            use_import_rules (bool): use import rule filters when loading the records [Default: True].

            filters (dict): use additional selection import filters.

            import_rules (dict): use custom import rules to import the records from the database.

            database (str): load records from the provided database.

        Returns:
            import_df (DataFrame): dataframe of imported records.
        """
        import_rules = self.import_rules if not import_rules else import_rules
        db = self._set_database(database)

        if isinstance(id_list, str):
            record_ids = [id_list]
        else:
            record_ids = id_list

        record_ids = sorted(list(set(record_ids)))  # prevents duplicate IDs
        logger.debug('loading records {IDS} of type "{TYPE}" from the database {DB}'
                     .format(IDS=record_ids, TYPE=self.name, DB=db))

        custom_filters = self._format_filter_set(filters) if filters else []

        # Add configured import filters
        table_statement = mod_db.format_tables(import_rules)
        columns = self._format_import_columns(import_rules)
        id_col = mod_db.get_import_column(import_rules, self.id_column)

        # Query existing database entries
        import_df = pd.DataFrame()
        for i in range(0, len(record_ids), 1000):  # split into sets of 1000 to prevent max parameter errors in SQL
            sub_ids = record_ids[i: i + 1000]
            if use_import_rules:
                filter_rules = custom_filters + mod_db.format_import_filters(import_rules)
            else:
                filter_rules = custom_filters

            filter_clause = '{COL} IN ({VALS})'.format(COL=id_col, VALS=','.join(['?' for _ in sub_ids]))
            filter_rules.append((filter_clause, tuple(sub_ids)))

            query = mod_db.prepare_sql_query(table_statement, columns=columns, filter_rules=filter_rules, order=id_col)
            loaded_df = user.read_db(*query, database=db)

            if import_df.empty:
                import_df = loaded_df
            else:
                import_df = import_df.append(loaded_df, ignore_index=True)

        logger.debug('{NLOADED} records passed the query filters out of {NTOTAL} requested records'
                     .format(NLOADED=import_df.shape[0], NTOTAL=len(record_ids)))

        return import_df

    def import_records(self, filter_params=None, filter_rules=None, import_rules: dict = None, database: str = None):
        """
        Import entry records from the database.

        Arguments:
            filter_params (list): list of parameter objects that will be used to filter the database when importing
                records.

            filter_rules (tuple): tuple or list of tuples containing where clause and value tuple for a given filter
                rule.

            import_rules (dict): use custom import rules to import the records from the database.

            database (str): load records from the provided database.
        """
        if isinstance(filter_params, list) or isinstance(filter_params, tuple):
            params = filter_params
        elif isinstance(filter_params, type(None)):
            params = []
        else:
            params = [filter_params]

        import_rules = self.import_rules if not import_rules else import_rules
        db = self._set_database(database)

        # Add configured import filters
        filters = mod_db.format_import_filters(import_rules)
        table_statement = mod_db.format_tables(import_rules)
        columns = self._format_import_columns(import_rules)
        #columns = mod_db.format_import_columns(import_rules)
        id_col = mod_db.get_import_column(import_rules, self.id_column)

        # Add optional parameter-based filters
        for param in params:
            dbcol = mod_db.get_import_column(import_rules, param.name)
            if dbcol:
                param_filter = param.query_statement(dbcol)
                if param_filter is not None:
                    filters.append(param_filter)

        # Add optional filters in tuple form
        if isinstance(filter_rules, list):
            filters.extend(filter_rules)
        elif isinstance(filter_rules, tuple):
            filters.append(filter_rules)

        # Query existing database entries
        query = mod_db.prepare_sql_query(table_statement, columns=columns, filter_rules=filters, order=id_col)
        import_df = user.read_db(*query, database=db)

        return import_df

    def import_references(self, ids, rule: str = None, filter_rules: list = None, is_reference: bool = False,
                          include_deleted: bool = False):
        """
        Import reference entries for the records associations.

        Arguments:
            ids (list): list of IDs to extract from the reference table. Default is to assume the list is a list of
                record IDs unless the is_reference argument is set to True.

            rule (str): name of the association rule to use to gather information about the references to extract
                [Default: import associations for all rules].

            filter_rules (list): list of tuples containing where clause and value tuple for a given filter rule.

            is_reference (bool): the provided list of IDs is a list of reference IDs.

            include_deleted (bool): import reference entries that were set to deleted as well.
        """
        association_rules = self.association_rules

        if rule and rule in association_rules:
            selected_types = [rule]
        else:
            selected_types = list(association_rules)

        # Remove duplicate IDs from the provided list of record IDs, if any
        if isinstance(ids, list):
            id_list = list(set(ids))
        elif isinstance(ids, str):
            id_list = [ids]
        elif isinstance(ids, pd.Series):
            id_list = ids.unique().tolist()
        else:
            id_list = []

        # Add optional filters in tuple form
        if isinstance(filter_rules, list):
            filter_set = filter_rules
        elif isinstance(filter_rules, tuple):
            filter_set = [filter_rules]
        else:
            filter_set = []

        # Prepare the import data
        references = {}
        for association_name in selected_types:
            try:
                assoc_entry = association_rules[association_name]
            except KeyError:
                msg = 'association rule {RULE} not found in the set of association rules for the record group' \
                    .format(RULE=association_name)
                logger.exception('RecordGroup {NAME}: {MSG}'.format(NAME=self.name, MSG=msg))

                raise ImportError(msg)

            is_primary = assoc_entry['Primary']
            reference_table = assoc_entry['ReferenceTable']

            if is_primary:  # input records are the primary record IDs
                columns = ['DocNo AS RecordID', 'RefNo AS ReferenceID', 'RefDate AS ReferenceDate',
                           'DocType AS RecordType',
                           'RefType AS ReferenceType', 'Notes AS ReferenceNotes', 'Warnings AS ReferenceWarnings',
                           'IsChild', 'IsHardLink', 'IsApproved']
                filter_str = 'DocNo IN ({VALS})' if is_reference is False else 'RefNo IN ({VALS})'
            else:  # input records are the reference record ID
                columns = ['DocNo AS ReferenceID', 'RefNo AS RecordID', 'RefDate AS ReferenceDate',
                           'DocType AS ReferenceType', 'RefType AS RecordType', 'Notes AS ReferenceNotes',
                           'Warnings AS ReferenceWarnings', 'IsChild', 'IsHardLink', 'IsApproved']
                filter_str = 'RefNo IN ({VALS})' if is_reference is False else 'DocNo IN ({VALS})'

            # Import reference entries related to record_id
            df = pd.DataFrame(columns=['RecordID', 'ReferenceID', 'ReferenceDate', 'RecordType', 'ReferenceType',
                                       'ReferenceNotes', 'ReferenceWarnings', 'IsChild', 'IsHardLink', 'IsApproved',
                                       'IsDeleted'])
            for i in range(0, len(id_list), 1000):  # split into sets of 1000 to prevent max parameter errors in SQL
                sub_ids = id_list[i: i + 1000]
                sub_vals = ','.join(['?' for _ in sub_ids])

                filters = [i for i in filter_set] + [(filter_str.format(VALS=sub_vals), tuple(sub_ids))]
                if not include_deleted:
                    filters.append(('IsDeleted = ?', 0))

                import_df = user.read_db(*mod_db.prepare_sql_query(reference_table, columns=columns,
                                                                   filter_rules=filters), prog_db=True)
                df = df.append(import_df, ignore_index=True)

            # Set column data types
            bool_columns = ['IsChild', 'IsHardLink', 'IsApproved', 'IsDeleted']
            df.fillna({i: False for i in bool_columns}, inplace=True)
            df = df.astype({i: np.bool_ for i in bool_columns})

            references[association_name] = df

        if len(selected_types) == 1:
            return references[selected_types[0]]
        else:
            return references

    def load_unreferenced_records(self, rule_name, database: str = None):
        """
        Load entry records that do not have a record reference for the given association rule.
        """
        association_rules = self.association_rules
        import_rules = self.import_rules
        db = self._set_database(database)

        try:
            rule = association_rules[rule_name]
        except KeyError:
            msg = 'association rule {RULE} not found in the set of association rules for the record group' \
                .format(RULE=rule_name)
            logger.exception('RecordGroup {NAME}: {MSG}'.format(NAME=self.name, MSG=msg))

            raise ImportError(msg)

        reference_table = rule['ReferenceTable']
        is_primary = rule['Primary']
        if is_primary:  # records are the primary in the reference table
            import_col_map = {'RecordID': 'DocNo', 'ReferenceID': 'RefNo', 'Deleted': 'IsDeleted'}
        else:  # reference record is the primary record ID
            import_col_map = {'RecordID': 'RefNo', 'ReferenceID': 'DocNo', 'Deleted': 'IsDeleted'}

        # Import reference entries related to record_id
        db_id_col = mod_db.get_import_column(import_rules, 'RecordID')
        #columns = mod_db.format_import_columns(import_rules)
        columns = self._format_import_columns(import_rules)
        filters = mod_db.format_import_filters(import_rules)
        # filters.append(('{TBL}.{COL} = ?'.format(TBL=reference_table, COL=import_col_map['Deleted']), 0))
        filters_clause = '{TBL}.{COL} IS NULL'.format(TBL=reference_table, COL=import_col_map['ReferenceID'])
        filters.append(filters_clause)

        join_statement = "{COL} = {TBL}.{REFCOL} AND {TBL}.{DELCOL} = '0'" \
            .format(COL=db_id_col, TBL=reference_table, REFCOL=import_col_map['RecordID'],
                    DELCOL=import_col_map['Deleted'])
        import_rules[reference_table] = {'Columns': {import_col_map['RecordID']: import_col_map['RecordID']},
                                         'Join': ["LEFT JOIN", join_statement]}

        table_statement = mod_db.format_tables(import_rules)
        import_df = user.read_db(*mod_db.prepare_sql_query(table_statement, columns=columns, filter_rules=filters),
                                 database=db)

        return import_df

    def confirm_saved(self, id_list, id_field: str = 'RecordID', table: str = None, database: str = None):
        """
        Check whether or not records have already been saved to the database.

        Arguments:
            id_list (list): check to see if these record IDs are found in the database or the given database table.

            id_field (str): table field containing the record IDs [Default: RecordID].

            table (str): search this database table for the record IDs.

            database (str): search this database for the record IDs.

        Returns:
            records_saved (list): list of corresponding booleans giving the status of the record ID in the database.
        """
        if isinstance(id_list, str):
            record_ids = [id_list]
        elif isinstance(id_list, pd.Series):
            record_ids = id_list.tolist()
        else:
            record_ids = id_list

        if not len(id_list) > 0:
            return []

        record_ids = sorted(list(set(record_ids)))  # prevents duplicate IDs
        logger.debug('verifying whether records {IDS} of type "{TYPE}" have been previously saved to the database'
                     .format(IDS=record_ids, TYPE=self.name))

        # Add configured import filters
        if table is None:
            db = settings.prog_db
            table_statement = mod_db.format_tables(self.import_rules)
            id_col = mod_db.get_import_column(self.import_rules, id_field)
            if not id_col:
                id_col = id_field
        else:
            db = self._set_database(database)
            table_statement = table
            id_col = id_field

        # Query existing database entries
        import_df = pd.DataFrame()
        for i in range(0, len(record_ids), 1000):  # split into sets of 1000 to prevent max parameter errors in SQL
            sub_ids = record_ids[i: i + 1000]
            filter_clause = '{COL} IN ({VALS})'.format(COL=id_col, VALS=','.join(['?' for _ in sub_ids]))
            filters = (filter_clause, tuple(sub_ids))

            if import_df.empty:
                import_df = user.read_db(*mod_db.prepare_sql_query(table_statement, columns=id_col,
                                                                   filter_rules=filters),
                                         database=db)
            else:
                import_df = import_df.append(user.read_db(*mod_db.prepare_sql_query(table_statement, columns=id_col,
                                                                                filter_rules=filters),
                                                          database=db),
                                             ignore_index=True)

        try:
            import_ids = import_df.iloc[:, 0].values.tolist()
        except IndexError as e:
            msg = 'failed to verify whether records {IDS} of type "{TYPE}" have been previously saved to the ' \
                  'database - {ERR}'.format(IDS=record_ids, TYPE=self.name, ERR=e)
            logger.error(msg)
            raise

        records_saved = pd.Series(index=record_ids, dtype='bool')
        for record_id in record_ids:
            if record_id in import_ids:
                records_saved[record_id] = True
            else:
                records_saved[record_id] = False

        if isinstance(id_list, str):
            return records_saved[id_list]
        else:
            return records_saved

    def save_database_references(self, ref_data, rule_name, statements: dict = None):
        """
        Prepare to save database references.

        Arguments:
            ref_data (DataFrame): reference entries to save to the database table specified by the association rule.

            rule_name (str): name of the association rule that indicates which database table to save the reference
                entries to.

            statements (dict): optional dictionary of transactions statements to append to.

        Returns:
            statements (dict): dictionary of transactions statements.
        """
        if statements is None:
            statements = {}

        if isinstance(ref_data, pd.DataFrame):
            df = ref_data
        elif isinstance(ref_data, pd.Series):
            df = ref_data.to_frame().transpose()
        elif isinstance(ref_data, dict):
            df = pd.DataFrame(ref_data)
        else:
            raise ValueError('ref_data must be one of DataFrame, Series, or dictionary')

        association_rules = self.association_rules
        try:
            rule = association_rules[rule_name]
        except KeyError:
            msg = 'association rule {RULE} not found in the set of association rules for the record group' \
                .format(RULE=rule_name)
            logger.exception('RecordGroup {NAME}: {MSG}'.format(NAME=self.name, MSG=msg))

            raise ImportError(msg)

        reference_table = rule['ReferenceTable']
        is_primary = rule['Primary']
        if is_primary:  # input record is the primary record ID
            primary_col = 'RecordID'
            export_col_map = {'RecordID': 'DocNo', 'ReferenceID': 'RefNo', 'ReferenceDate': 'RefDate',
                              'RecordType': 'DocType', 'ReferenceType': 'RefType', 'ReferenceNotes': 'Notes',
                              'ReferenceWarnings': 'Warnings', 'IsApproved': 'IsApproved', 'IsChild': 'IsChild',
                              'IsHardLink': 'IsHardLink', 'IsDeleted': 'IsDeleted'}
        else:  # reference record is the primary record ID
            primary_col = 'ReferenceID'
            export_col_map = {'ReferenceID': 'DocNo', 'RecordID': 'RefNo', 'ReferenceDate': 'RefDate',
                              'ReferenceType': 'DocType', 'RecordType': 'RefType', 'ReferenceNotes': 'Notes',
                              'ReferenceWarnings': 'Warnings', 'IsApproved': 'IsApproved', 'IsChild': 'IsChild',
                              'IsHardLink': 'IsHardLink', 'IsDeleted': 'IsDeleted'}

        # Remove rows where the primary column is NULL
        df.drop(df[df[primary_col].isna()].index, inplace=True)
        if df.empty:
            logger.warning('RecordType {NAME}: no reference entries provided for saving'.format(NAME=self.name))
            return statements

        # Prepare separate update and insert statements depending on whether an individual reference entry exists
        export_df = df[[i for i in export_col_map if i in df.columns]].rename(columns=export_col_map)

        # Prepare the upsert statement
        export_columns = export_df.columns.tolist()
        export_values = [tuple(i) for i in export_df.values.tolist()]

        statements = mod_db.prepare_sql_upsert(reference_table, export_columns, export_values, ['DocNo', 'RefNo'],
                                               statements=statements)

        return statements

    def delete_database_references(self, ref_data, rule_name, statements: dict = None):
        """
        Prepare to delete database reference entries from the database.

        Arguments:
            ref_data (DataFrame): reference entries to delete from the database table specified by the association rule.

            rule_name (str): name of the association rule linking the relevant records.

            statements (dict): optional dictionary of transactions statements to append to.

        Returns:
            statements (dict): dictionary of transactions statements.
        """
        if statements is None:
            statements = {}

        if isinstance(ref_data, pd.DataFrame):
            df = ref_data
        elif isinstance(ref_data, pd.Series):
            df = ref_data.to_frame().transpose()
        elif isinstance(ref_data, dict):
            df = pd.DataFrame(ref_data)
        else:
            raise ValueError('ref_data must be one of DataFrame, Series, or dictionary')

        if df.empty:
            logger.warning('RecordGroup {NAME}: no reference entries provided for deleting'.format(NAME=self.name))
            return statements

        association_rules = self.association_rules
        try:
            rule = association_rules[rule_name]
        except KeyError:
            msg = 'association rule {RULE} not found in the set of association rules for the record group' \
                .format(RULE=rule_name)
            logger.exception('RecordGroup {NAME}: {MSG}'.format(NAME=self.name, MSG=msg))

            raise ImportError(msg)

        reference_table = rule['ReferenceTable']
        is_primary = rule['Primary']
        if is_primary:  # input record is the primary record ID
            export_col_map = {'RecordID': 'DocNo', 'ReferenceID': 'RefNo', 'IsDeleted': 'IsDeleted'}
        else:  # reference record is the primary record ID
            export_col_map = {'ReferenceID': 'DocNo', 'RecordID': 'RefNo', 'IsDeleted': 'IsDeleted'}

        # Prepare the export reference entries
        df = df[(~df['RecordID'].isna()) & (~df['ReferenceID'].isna())]
        export_df = df[[i for i in export_col_map if i in df.columns]].rename(columns=export_col_map)

        # Prepare the update statement
        export_columns = [export_col_map['IsDeleted']]
        export_values = [(1,) for _ in range(export_df.shape[0])]

        filter_clause = '{COL} = ? AND {REFCOL} = ?'.format(COL=export_col_map['RecordID'],
                                                            REFCOL=export_col_map['ReferenceID'])
        filter_params = [(row[export_col_map['RecordID']], row[export_col_map['ReferenceID']]) for _, row in
                         export_df.iterrows()]

        statements = mod_db.prepare_sql_update(reference_table, export_columns, export_values, filter_clause,
                                               filter_params, statements=statements)

        return statements

    def save_database_records(self, records, id_field: str = 'RecordID', statements: dict = None,
                              export_columns: bool = True, ref_ids: list = []):
        """
        Create insert and update database transaction statements for records.

        Arguments:
            records (DataFrame): record data that will be exported to the database.

            id_field (str): name of the column containing the record IDs.

            statements (dict): optional dictionary of database transaction statements to append to.

            export_columns (bool): use import column mapping to transform column names to database names before
                exporting [Default: True].

            ref_ids (list): list of references that have already been modified within the current save action.
                Required to prevent endless recursion.

        Returns:
            statements (dict): dictionary of transactions statements.
        """
        # pd.set_option('display.max_columns', None)
        export_rules = self.export_rules
        association_rules = self.association_rules

        if not statements:
            statements = {}

        if not self.program_record:
            msg = 'unable to modify database records for record group {NAME} - {NAME} is an external record group'\
                .format(NAME=self.name)
            logger.warning(msg)

            return statements

        save_time = datetime.datetime.now().strftime(settings.date_format)

        if isinstance(records, pd.DataFrame):
            df = records
        elif isinstance(records, pd.Series):
            df = records.to_frame().transpose()
        elif isinstance(records, dict):
            df = pd.DataFrame(records)
        else:
            raise ValueError('records argument must be one of DataFrame, Series, or dictionary')

        if df.empty:
            logger.warning('RecordType {NAME}: no records provided for saving'.format(NAME=self.name))
            return statements

        exists = self.confirm_saved(df[id_field].values.tolist(), id_field=id_field)
        existing_ids = exists[exists].index.tolist()
        new_ids = exists[~exists].index.tolist()

        # Prepare a separate database transaction statement for each database table containing the record's data
        columns = df.columns.tolist()
        for table in export_rules:
            table_entry = export_rules[table]

            if export_columns:
                export_col_map = table_entry['Columns']
            else:
                export_col_map = {i: i for i in table_entry['Columns'].values()}

            try:
                id_col = export_col_map[id_field]
            except KeyError:
                msg = 'RecordGroup {NAME}: missing ID column "{COL}" from record import columns {COLS}' \
                    .format(NAME=self.name, COL=id_field, COLS=list(export_col_map.keys()))
                logger.error(msg)

                raise KeyError(msg)

            # Prepare column value updates
            include_columns = [i for i in columns if i in export_col_map]
            export_df = df[include_columns]

            # Prepare separate update and insert statements depending on whether an individual record already exists

            # Extract all currently existing records from the table
            current_df = export_df[export_df[id_field].isin(existing_ids)]

            # Prepare update statements for the existing records
            if not current_df.empty:
                # Add edit details to records table
                current_df.loc[:, settings.editor_code] = user.uid
                current_df.loc[:, settings.edit_date] = save_time

                export_columns = current_df.rename(columns=export_col_map).columns.tolist()
                export_values = [tuple(i) for i in current_df.values.tolist()]

                record_ids = current_df[id_field]
                if not isinstance(record_ids, pd.Series):
                    record_ids = [record_ids]
                else:
                    record_ids = record_ids.values.tolist()
                filter_params = [(i,) for i in record_ids]
                filter_clause = '{COL} = ?'.format(COL=id_col)
                statements = mod_db.prepare_sql_update(table, export_columns, export_values, filter_clause,
                                                       filter_params, statements=statements)

            # Extract all new records from the table
            new_df = export_df[export_df[id_field].isin(new_ids)]

            # Prepare insertion statements for the new records
            if not new_df.empty:
                # Add record creation details to records table
                try:
                    new_df[settings.creator_code].fillna(user.uid, inplace=True)
                    new_df[settings.creation_date].fillna(save_time, inplace=True)
                except KeyError:
                    new_df.loc[:, settings.creator_code] = user.uid
                    new_df.loc[:, settings.creation_date] = save_time

                export_columns = new_df.rename(columns=export_col_map).columns.tolist()
                export_values = [tuple(i) for i in new_df.values.tolist()]
                statements = mod_db.prepare_sql_insert(table, export_columns, export_values, statements=statements)

        # If relevant, create or edit hard-linked reference records for new database records
        new_df = df[df[id_field].isin(new_ids)].rename(columns={id_field: 'RecordID'})
        exist_df = df[df[id_field].isin(existing_ids)].rename(columns={id_field: 'RecordID'})
        for association in association_rules:
            rule = association_rules[association]

            # Create or edit any hard-linked records
            link_rules = rule['HardLink']
            if link_rules is not None:
                record_type = self.name

                for ref_type in link_rules:
                    ref_entry = settings.records.fetch_rule(ref_type)
                    link_rule = link_rules[ref_type]

                    try:
                        condition = link_rule['Condition']
                    except KeyError:
                        condition = None
                    try:
                        colmap = link_rule['ColumnMap']
                    except KeyError:
                        msg = 'missing required HardLink parameter "ColumnMap"'
                        logger.error('RecordGroup {NAME}: {MSG}'.format(NAME=self.name, MSG=msg))

                        raise KeyError(msg)

                    sub_cols = [i for i in colmap if i in df.columns]

                    # Edit existing linked-records

                    # Get hard-linked references of the existing records
                    current_ids = exist_df['RecordID'].tolist()
                    exist_ref_df = self.import_references(current_ids, rule=association)
                    assoc_record_ids = exist_ref_df['RecordID'].tolist()

                    # Update hard-linked record element values to match record element values
                    exist_ref_df = exist_ref_df[
                        (exist_ref_df['IsHardLink']) & (~exist_ref_df['ReferenceID'].isin(ref_ids))]
                    if not exist_ref_df.empty:
                        # Merge the relevant columns of the records dataframe with the reference dataframe
                        merged_df = pd.merge(exist_df[['RecordID'] + sub_cols], exist_ref_df, on='RecordID')

                        # Subset the dataframe on the mapped columns and rename the columns
                        merged_df = merged_df[['ReferenceID'] + sub_cols].rename(columns=colmap)
                        merged_df.rename(columns={'ReferenceID': 'RecordID'}, inplace=True)

                        # Prepare the update statements for the hard-linked records
                        statements = ref_entry.save_database_records(merged_df, statements=statements,
                                                                     ref_ids=current_ids)

                    if not rule['Primary']:  # restricting to primary record types prevents endless recursion
                        continue

                    # Allow the user to delete dependent records when dependency conditions are no longer met
                    # if condition and not exist_df.empty:
                    #    delete_ids = exist_df.loc[~mod_dm.evaluate_condition(exist_df, condition), 'RecordID']
                    #    delete_ref_df = exist_ref_df[exist_ref_df['RecordID'].isin(delete_ids.tolist())]
                    #    to_delete = []
                    #    for index, row in delete_ref_df.iterrows():
                    #        delete_id = row['ReferenceID']
                    #        parent_id = row['RecordID']
                    #        msg = ''.format(DOCTYPE=self.menu_title, PID=parent_id, ID=delete_id, REFTYPE=ref_entry.menu_title)
                    #        confirm = mod_win2.popup_confirm(msg)
                    #        if confirm == 'OK':
                    #            to_delete.append(delete_id)

                    #    statements = ref_entry.delete_database_records(to_delete, statements=statements)

                    # Create new hard-linked records
                    create_df = new_df.copy()
                    if condition and not create_df.empty:
                        create_df = create_df[mod_dm.evaluate_condition(create_df, condition)]

                    # Add existing records w/o hard-links to the "add df" so that new hard-linked records can be created
                    missing_df = exist_df[~exist_df['RecordID'].isin(assoc_record_ids)]
                    if condition and not missing_df.empty:
                        missing_df = missing_df[mod_dm.evaluate_condition(missing_df, condition)]

                    for index, row in missing_df.iterrows():
                        msg = 'Existing {DOCTYPE} record {ID} does not currently have an associated {REFTYPE} record. ' \
                              'Would you like to create one now?'.format(DOCTYPE=self.menu_title, ID=row['RecordID'],
                                                                         REFTYPE=ref_entry.menu_title)
                        confirm = mod_win2.popup_confirm(msg)
                        if confirm == 'OK':
                            create_df = create_df.append(row, ignore_index=True)

                    if create_df.empty:
                        continue

                    # Create new record IDs for the hard-linked records
                    primary_ids = create_df['RecordID'].tolist()
                    try:
                        ref_dates = pd.to_datetime(create_df['RecordDate'], errors='coerce')
                    except KeyError:
                        msg = 'failed to create IDs for the new records - failed to create associated "{TYPE}" ' \
                              'records'.format(TYPE=record_type, RTYPE=ref_type)
                        logger.error('RecordGroup {NAME}: {MSG}'.format(NAME=self.name, MSG=msg))

                        raise KeyError(msg)
                    else:
                        ref_dates = ref_dates.tolist()

                    ref_ids = ref_entry.create_record_ids(ref_dates, offset=settings.get_date_offset())

                    # Subset the dataframe on the mapped columns and rename the columns
                    ref_df = create_df[sub_cols].rename(columns=colmap)
                    ref_df['RecordID'] = ref_ids
                    ref_df['RecordDate'] = ref_dates

                    # Create the hard-linked records
                    statements = ref_entry.save_database_records(ref_df, statements=statements)

                    # Create the record references
                    ref_data = pd.DataFrame({'RecordID': primary_ids, 'ReferenceID': ref_ids,
                                             'ReferenceDate': datetime.datetime.now(), 'RecordType': record_type,
                                             'ReferenceType': ref_type, 'IsApproved': True, 'IsChild': False,
                                             'IsHardLink': True, 'IsDeleted': False})
                    statements = self.save_database_references(ref_data, association, statements=statements)

        return statements

    def delete_database_records(self, records, statements: dict = None, id_field: str = 'RecordID',
                                ref_ids: list = None):
        """
        Delete records from the database.

        Arguments:
            records (list): delete records with these record IDs from the database.

            statements (dict): optional dictionary of database transaction statements to add to.

            id_field (str): name of the column containing the record IDs [Default: RecordID].

            ref_ids (list): list of references that have already been modified within the current delete action.
                Required to prevent endless recursion.

        Returns:
            statements (dict): dictionary of transaction statements.
        """
        # pd.set_option('display.max_columns', None)

        if not statements:
            statements = {}

        if not self.program_record:
            msg = 'unable to modify database records for record group {NAME} - {NAME} is an external record group'\
                .format(NAME=self.name)
            logger.warning(msg)

            return statements

        if ref_ids is None:
            ref_ids = []

        if isinstance(records, str):
            records = [records]
        elif isinstance(records, pd.Series):
            records = records.tolist()

        if not len(records) > 0:  # empty list provided
            return statements

        records = list(set(records))  # duplicate filtering

        # Check existence of the records in the database
        exists = self.confirm_saved(records, id_field=id_field)

        record_ids = []
        for record_id in records:
            record_exists = exists[record_id]
            if record_exists:  # only attempt to delete records that already exist in the database
                record_ids.append(record_id)

        if len(record_ids) < 1:  # no currently existing records to delete
            return statements

        # Set existing records as deleted in the database
        export_rules = self.export_rules
        for export_table in export_rules:
            table_entry = export_rules[export_table]

            export_col_map = table_entry['Columns']
            if 'Deleted' not in export_col_map:
                continue

            id_col = export_col_map[id_field]
            delete_col = export_col_map['Deleted']

            export_columns = [delete_col, settings.editor_code, settings.edit_date]
            export_values = [(1, user.uid, datetime.datetime.now().strftime(settings.date_format)) for _ in record_ids]

            filter_params = [(i,) for i in record_ids]
            filter_clause = '{COL} = ?'.format(COL=id_col)

            # Remove records from the export table
            statements = mod_db.prepare_sql_update(export_table, export_columns, export_values, filter_clause,
                                                   filter_params, statements=statements)

        # Remove record associations and potentially delete associated records if associated records are child records
        # or hard-linked to the deleted records
        association_rules = self.association_rules
        for association in association_rules:
            rule = association_rules[association]
            assoc_type = rule['AssociationType']

            # Import reference entries to be deleted
            import_df = self.import_references(record_ids, rule=association)

            # Delete all reference entries associated with the record IDs and remove already used references from the
            # list - this is necessary for hard-linked records to avoid endless looping
            # import_df['IsDeleted'] = True
            export_df = import_df.drop(import_df[import_df['ReferenceID'].isin(ref_ids)].index)
            if export_df.empty:
                continue
            # statements = self.save_database_references(export_df, association, statements=statements)
            statements = self.delete_database_references(export_df, association, statements=statements)

            # Subset reference entries to include those that are child records or hard-linked
            if assoc_type == 'parent':  # referenced records are child records and should be deleted with parent
                condition = export_df['IsChild'].astype(bool)
            elif assoc_type == 'reference':  # deleting hard-linked records should also delete reference records
                condition = export_df['IsHardLink'].astype(bool)
            else:  # record is a child record - deleted child records should have no affect on the parent records
                continue

            linked_df = export_df[condition]
            if linked_df.empty:
                continue

            # Update the list of used reference IDs
            ignore_ids = list(set(ref_ids + record_ids))

            # Remove hard-linked and child records. Do not include reference entries that have already been deleted.
            record_types = import_df['ReferenceType'].unique()
            for record_type in record_types:
                record_entry = settings.records.fetch_rule(record_type)
                if record_entry is None:
                    msg = 'unable to delete dependant records of record type "{TYPE}"' \
                        .format(TYPE=record_type)
                    logger.error('RecordGroup {NAME}: {MSG}'.format(NAME=self.name, MSG=msg))

                    raise AttributeError(msg)

                # Subset imported reference entries by record class of the reference record
                sub_df = linked_df[linked_df['ReferenceType'] == record_type]

                # Prepare deletion statements for reference records
                ref_ids = sub_df['ReferenceID'].values.tolist()
                statements = record_entry.delete_database_records(ref_ids, statements=statements, ref_ids=ignore_ids)

        return statements

    def _search_saved_ids(self, record_dates, id_field: str = 'RecordID'):
        """
        Get a list of saved record IDs for records with record date within the provided range of dates.
        """
        # Prepare query parameters
        table_statement = mod_db.format_tables(self.import_rules)
        id_col = mod_db.get_import_column(self.import_rules, id_field)

        # Prepare the date range
        sorted_dates = sorted(record_dates)
        try:
            first_date = sorted_dates[0]
            last_date = sorted_dates[-1]
        except IndexError:
            logger.error('failed to import saved record IDs - no dates provided to the method')
            raise

        first_day = first_date.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
        last_day = datetime.datetime(last_date.year + last_date.month // 12, last_date.month % 12 + 1, 1) - \
                   datetime.timedelta(1)

        # Connect to database
        params = (first_day, last_day)
        filters = ('{DATE} BETWEEN ? AND ?'.format(DATE=settings.date_field), params)
        import_rows = user.read_db(*mod_db.prepare_sql_query(table_statement, columns=id_col, filter_rules=filters,
                                                             order=id_col),
                                   prog_db=self.program_record)

        try:
            id_list = import_rows.iloc[:, 0]
        except IndexError:
            logger.info('no existing record IDs found')
            record_ids = []
        except Exception as e:
            logger.error('failed to import saved record IDs - {ERR}'.format(ERR=e))
            raise
        else:
            record_ids = id_list.values.tolist()

        return record_ids

    def create_record_ids(self, date_list, offset: int = 0):
        """
        Create a new set of record IDs.

        Arguments:
            date_list (list): create IDs from these dates.

            offset (int): apply an offset to the date list that will be used within the new record IDs by this amount
                [Default: do not apply a date offset].

        Returns:
            record_ids (list): list of new, unique record IDs.
        """
        relativedelta = dateutil.relativedelta.relativedelta
        is_datetime_dtype = pd.api.types.is_datetime64_any_dtype
        strptime = datetime.datetime.strptime

        record_type = self.name
        id_code = self.id_code

        if isinstance(date_list, str) or is_datetime_dtype(type(date_list)) or isinstance(date_list, datetime.datetime):
            record_dates = [date_list]
            single_value = True
        elif isinstance(date_list, list) or isinstance(date_list, tuple):
            if len(date_list) < 1:
                return []

            record_dates = date_list
            single_value = False
        else:
            logger.error('failed to create IDs for the record entries of type "{TYPE}" - record_dates must be '
                         'formatted as a list, string, or datetime object not {OTYPE}'
                         .format(TYPE=record_type, OTYPE=type(date_list)))
            return None

        logger.info('creating {N} new record IDs for records of type "{TYPE}"'
                    .format(N=len(record_dates), TYPE=record_type))

        # Get list of unsaved record IDs of the same record type
        record_dates = mod_dm.format_values(record_dates, 'datetime')
        unsaved_ids = self.get_unsaved_ids(internal_only=False)
        if unsaved_ids is None:
            logger.error('failed to create IDs for the record entries of type "{TYPE}" - unable to obtain a list of '
                         'unsaved record IDs'.format(TYPE=record_type))
            return None

        # Get list of saved record IDs of the same record type within the range of provided dates
        try:
            saved_ids = self._search_saved_ids(record_dates)
        except Exception as e:
            logger.exception('failed to create IDs for the record entries of type {TYPE} from the set of date {DATE} - '
                             '{ERR}'.format(TYPE=record_type, DATE=record_dates, ERR=e))
            return None

        # Format the date component of the new ID
        record_ids = []
        for record_date in record_dates:
            try:
                id_date = (record_date + relativedelta(years=+offset)).strftime(
                    settings.format_date_str(date_str='YYMM'))
            except Exception as e:
                logger.debug(e)
                id_date = (strptime(record_date.strftime('%Y-%m-%d %H:%M:%S'), '%Y-%m-%d %H:%M:%S')
                           + relativedelta(years=+offset)).strftime(settings.format_date_str(date_str='YYMM'))

            logger.debug('RecordGroup {NAME}: new ID has date component {COMP}'.format(NAME=record_type, COMP=id_date))

            # Search list of unsaved IDs occurring within the current date cycle
            logger.debug('RecordGroup {NAME}: searching for unsaved record IDs with date component {DATE}'
                         .format(NAME=record_type, DATE=id_date))
            prev_ids = []
            for unsaved_id in unsaved_ids:
                prev_date = self._id_date_component(unsaved_id)
                if prev_date == id_date:
                    prev_ids.append(unsaved_id)

            logger.debug('RecordGroup {NAME}: found {NUM} unsaved records with date component {DATE}'
                         .format(NAME=record_type, NUM=len(prev_ids), DATE=id_date))

            # Search list of saved IDs occurring within the current date cycle
            logger.debug('RecordGroup {NAME}: searching for database record IDs with date component {DATE}'
                         .format(NAME=record_type, DATE=id_date))

            for saved_id in saved_ids:
                prev_date = self._id_date_component(saved_id)
                if prev_date == id_date:
                    prev_ids.append(saved_id)

            # Get the number of the last ID used in the current date cycle
            logger.debug('RecordGroup {NAME}: found {NUM} records with date component {DATE}'
                         .format(NAME=self.name, NUM=len(prev_ids), DATE=id_date))

            if len(prev_ids) > 0:
                prev_ids.sort()
                last_id = prev_ids[-1]
            else:
                last_id = None

            # Create the new ID
            if last_id:
                logger.debug('RecordGroup {NAME}: last ID encountered is {ID}'.format(NAME=record_type, ID=last_id))
                try:
                    last_num = int(last_id.split('-')[-1])
                except ValueError:
                    msg = 'RecordGroup {NAME}: incorrect formatting for previous ID {ID}' \
                        .format(NAME=record_type, ID=last_id)
                    logger.error(msg)
                    record_ids.append(None)
                    continue
            else:
                logger.debug('RecordGroup {NAME}: no previous IDs found for date {DATE} - starting new iteration at 1'
                             .format(NAME=record_type, DATE=id_date))
                last_num = 0

            record_id = '{CODE}{DATE}-{NUM}'.format(CODE=id_code, DATE=id_date, NUM=str(last_num + 1).zfill(4))

            logger.info('RecordGroup {NAME}: new record ID is {ID}'.format(NAME=record_type, ID=record_id))
            record_ids.append(record_id)
            unsaved_ids.append(record_id)

        failed_rows = [i + 1 for i, j in enumerate(record_ids) if not j]
        if len(failed_rows) > 0:
            msg = 'failed to create record IDs for table entries at rows {ROW}'.format(ROW=failed_rows)
            logger.error(msg)

            return None

        success = self.add_unsaved_ids(record_ids)
        if not success:
            logger.error('failed to create IDs for the record entries of type "{TYPE}" - unable to add record IDs to '
                         'the list unsaved record IDs'.format(TYPE=record_type))
            return None

        if single_value:
            return record_ids[0]
        else:
            return record_ids

    def _id_date_component(self, record_id):
        """
        Get the date component of a record ID.
        """
        id_code = self.id_code
        code_len = len(id_code)
        try:
            id_name, id_num = record_id.split('-')
        except ValueError:
            raise AttributeError('no date component found for {TYPE} record ID {ID}'
                                 .format(TYPE=self.name, ID=record_id))

        return id_name[code_len:]

    def remove_unsaved_ids(self, record_ids: list = None, internal_only: bool = True):
        """
        Remove a record ID from the database of unsaved IDs associated with the record type.

        Arguments:
            record_ids (list): optional list of record IDs to remove from the set of unsaved record IDs with ID code
                corresponding to the ID code configured for the record group.

            internal_only (bool): only remove recordIDs from the unsaved record IDs if they were created within the
                program instance.

        Returns:
            success (bool): removal of unsaved record IDs was successful.
        """
        unsaved_ids = self.get_unsaved_ids(internal_only=internal_only)

        if isinstance(record_ids, str):
            remove_ids = [record_ids] if record_ids in unsaved_ids else []
        elif isinstance(record_ids, list):
            remove_ids = [i for i in record_ids if i in unsaved_ids]
        else:
            remove_ids = unsaved_ids

        if not len(record_ids) > 0:
            logger.debug('RecordGroup {NAME}: no unsaved record IDs to remove'.format(NAME=self.name))
            return True

        logger.debug('RecordGroup {NAME}: attempting to remove IDs {ID} from the list of unsaved record IDs'
                     .format(NAME=self.name, ID=remove_ids))

        value = {'ids': remove_ids, 'id_code': self.id_code}
        content = {'action': 'remove_ids', 'value': value}
        request = {'content': content, 'encoding': "utf-8"}
        response = server_conn.process_request(request)

        success = response['success']
        if success is False:
            msg = 'RecordGroup {NAME}: failed to remove IDs {ID} from the list of unsaved record IDs of type ' \
                  '{TYPE} - {ERR}'.format(NAME=self.name, ID=remove_ids, TYPE=self.name, ERR=response['value'])
            logger.error(msg)
        else:
            logger.debug('RecordGroup {NAME}: successfully removed {ID} from the list of unsaved record IDs '
                         'associated with the record group'.format(NAME=self.name, ID=remove_ids))

        return success

    def get_unsaved_ids(self, internal_only: bool = False):
        """
        Retrieve a list of unsaved record IDs from the database of unsaved record IDs associated with the record type.

        Arguments:
            internal_only (bool): only retrieve unsaved record IDs that they were created within the current program
                instance.

        Returns:
            unsaved_ids (list): list of unsaved record IDs.
        """
        if internal_only is True:
            instance = settings.instance_id
            logger.debug('RecordGroup {NAME}: attempting to obtain an instance-specific list of unsaved record IDs '
                         'associated with the record group'.format(NAME=self.name))
        else:
            instance = None
            logger.debug('RecordGroup {NAME}: attempting to obtain list of unsaved record IDs associated with the '
                         'record group'.format(NAME=self.name))

        value = {'instance': instance, 'id_code': self.id_code}
        content = {'action': 'request_ids', 'value': value}
        request = {'content': content, 'encoding': "utf-8"}
        response = server_conn.process_request(request)

        if response['success'] is False:
            msg = 'failed to obtain list of unsaved record IDs of type {TYPE} from the server - {ERR}' \
                .format(TYPE=self.name, ERR=response['value'])
            logger.error(msg)

            return []
        else:
            logger.debug('RecordGroup {NAME}: successfully obtained a list of unsaved record IDs associated with the '
                         'record group'.format(NAME=self.name))

        unsaved_ids = response['value']

        return unsaved_ids

    def add_unsaved_ids(self, record_ids):
        """
        Add a record ID to the list of unsaved record IDs associated with the record type.

        Arguments:
            record_ids (list): list of record IDs to add to the set of unsaved record IDs with ID code corresponding to
                the ID code of the record group.

        Returns:
            success (bool): addition of records to the set of unsaved record IDs was successful.
        """
        logger.debug('RecordGroup {NAME}: attempting to add record IDs {ID} to the list of unsaved record IDs '
                     'associated with the record group'.format(NAME=self.name, ID=record_ids))

        if isinstance(record_ids, str):
            id_set = [(record_ids, settings.instance_id)]
        else:
            id_set = [(i, settings.instance_id) for i in record_ids]

        value = {'ids': id_set, 'id_code': self.id_code}
        content = {'action': 'add_ids', 'value': value}
        request = {'content': content, 'encoding': "utf-8"}
        response = server_conn.process_request(request)

        success = response['success']
        if success is False:
            msg = 'failed to add {ID} to the list of unsaved record IDs of type {TYPE} on the server - {ERR}' \
                .format(NAME=self.name, ID=record_ids, TYPE=self.name, ERR=response['value'])
            logger.error(msg)
        else:
            logger.debug('RecordGroup {NAME}: successfully added record IDs {ID} to the list of unsaved record IDs '
                         'associated with the record group'.format(NAME=self.name, ID=record_ids))

        return success

class DatabaseRecord:
    """
    Generic database record account.

    Attributes:
        name (str): name of the record's record group.

        id (int): record element number.

        elements (list): list of GUI element keys.

        title (str): record display title.

        components (list): list of all record elements.

        tabs (dict): describes how layout sections are divided into tabs.

        sections (dict): describes how record elements are divided into sections.

        report (dict): report definition.
    """

    def __init__(self, name, entry, level: int = 0):
        """
        Arguments:
            name (str): name of the record's record group (record type).

            entry (class): configuration entry for the record layout.

            level (int): depth at which record was opened [Default: 0].
        """
        # Reserved fields
        self.id_field = 'RecordID'
        self.date_field = 'RecordDate'
        self.delete_field = 'Deleted'

        # Record properties
        self.new = False
        self.level = level

        self.name = name

        self.id = randint(0, 1000000000)
        self.elements = {i: '-{NAME}_{ID}_{ELEM}-'.format(NAME=self.name, ID=self.id, ELEM=i) for i in
                         ('Element', 'TG', 'Header')}
        self.bindings = {}

        try:
            self.title = entry['Title']
        except KeyError:
            self.title = self.name

        # Header elements
        try:
            headers = entry['Header']
        except KeyError:
            headers = {}

        try:
            id_entry = headers[self.id_field]
        except KeyError:
            id_entry = {'ElementType': 'text', 'DataType': 'varchar', 'Description': 'Record ID'}
        else:
            id_entry['Modifiers'] = {}
            id_entry['ElementType'] = 'text'
            id_entry['DataType'] = 'varchar'
        try:
            self._record_id = mod_elem.DataConstant(self.id_field, id_entry)
        except Exception as e:
            logger.error('RecordType {NAME}: {MSG}'.format(NAME=self.name, MSG=e))
            raise AttributeError(e)

        try:
            date_entry = headers[self.date_field]
        except KeyError:
            date_entry = {'ElementType': 'text', 'DataType': 'date', 'Description': 'Record Date'}
        else:
            date_entry['Modifiers'] = {}
            date_entry['ElementType'] = 'text'
            date_entry['DataType'] = 'date'
        try:
            self._record_date = mod_elem.DataConstant(self.date_field, date_entry)
        except Exception as e:
            logger.error('RecordType {NAME}: {MSG}'.format(NAME=self.name, MSG=e))
            raise AttributeError(e)

        # Component record elements
        self.components = []
        self.blanks = []
        used_elements = []

        try:
            record_elements = entry['RecordElements']
        except KeyError:
            msg = 'missing configuration parameter "RecordElements"'
            logger.error('RecordGroup {NAME}: {MSG}'.format(NAME=self.name, MSG=msg))
        else:
            for element_name in record_elements:
                if element_name in used_elements:
                    msg = 'more than one entry configured for record element {ELEM}'.format(ELEM=element_name)
                    logger.error('RecordGroup {NAME}: {MSG}'.format(NAME=self.name, MSG=msg))

                    continue
                else:
                    used_elements.append(element_name)

                # Initialize the record element
                elem_entry = record_elements[element_name]
                try:
                    elem_obj = initialize_element(self.name, element_name, elem_entry)
                except Exception as e:
                    raise AttributeError('failed to initialize {NAME} element {ELEM} - {ERR}'
                                         .format(NAME=self.name, ELEM=element_name, ERR=e))

                # Add the element to the set of record components
                self.components.append(elem_obj)

        # Record layout attributes
        self.tabs = {}
        used_sections = []
        try:
            tabs = entry['Tabs']
        except KeyError:
            msg = 'missing configuration parameter "Tabs"'
            logger.warning('RecordGroup {NAME}: {MSG}'.format(NAME=self.name, MSG=msg))

            tabs = {}

        for tab_name in tabs:
            _tab_entry = tabs[tab_name]
            tab_entry = {}

            try:
                tab_entry['Title'] = _tab_entry['Title']
            except KeyError:
                tab_entry['Title'] = tab_name

            tab_sections = []
            try:
                _tab_sections = _tab_entry['Sections']
            except KeyError:
                msg = 'no sections defined for tab layout {TAB}'.format(TAB=tab_name)
                logger.warning('RecordGroup {NAME}: {MSG}'.format(NAME=self.name, MSG=msg))

                continue
            else:
                for tab_section in _tab_sections:
                    if tab_section in used_sections:
                        msg = 'unable to add section {SEC} to tab layout {TAB} - section {SEC} is already defined ' \
                              'for a previous tab'.format(TAB=tab_name, SEC=tab_section)
                        logger.warning('RecordGroup {NAME}: {MSG}'.format(NAME=self.name, MSG=msg))

                        continue

                    tab_sections.append(tab_section)
                    used_sections.append(tab_section)

            if len(tab_sections) > 0:
                tab_entry['Sections'] = tab_sections
                self.tabs[tab_name] = tab_entry

        self.sections = {}
        used_section_elements = []
        try:
            sections = entry['Sections']
        except KeyError:
            msg = 'missing configuration parameter "Sections"'
            logger.warning('RecordGroup {NAME}: {MSG}'.format(NAME=self.name, MSG=msg))

            sections = {}

        for i, section in enumerate(sections):
            _section_entry = sections[section]
            section_entry = {}

            try:
                section_entry['Title'] = _section_entry['Title']
            except KeyError:
                section_entry['Title'] = section

            if 'HeadingElement' in _section_entry:
                heading_element = _section_entry['HeadingElement']
                used_section_elements.append(heading_element)
                section_entry['HeadingElement'] = heading_element

            try:
                _section_layout = _section_entry['Layout']
            except KeyError:
                msg = 'no record elements defined for section layout {SEC}'.format(SEC=section)
                logger.warning('RecordGroup {NAME}: {MSG}'.format(NAME=self.name, MSG=msg))

                continue
            else:
                section_layout = {}
                row_num = 0
                for section_row in _section_layout:
                    _row_elements = _section_layout[section_row]
                    if not isinstance(_row_elements, list):
                        _row_elements = [_row_elements]
                    row_len = len(_row_elements)

                    row_elements = []
                    remaining_elements = []
                    column_num = 0
                    for element_name in _row_elements:
                        if element_name in used_section_elements:
                            msg = 'unable to add record element {ELEM} to section layout {SEC} - element {ELEM} is ' \
                                  'already defined in a previous section'.format(ELEM=element_name, SEC=section)
                            logger.warning('RecordGroup {NAME}: {MSG}'.format(NAME=self.name, MSG=msg))

                            continue

                        if element_name == '' and column_num < 3:
                            blank_name = 'Blank{SEC}.{ROW}.{COL}'.format(SEC=i, ROW=row_num, COL=column_num)
                            blank_element = mod_elem.BlankField(blank_name, parent=self.name)

                            self.blanks.append(blank_element)
                            row_elements.append(blank_name)

                            column_num += 1

                            continue

                        try:
                            section_element = self.fetch_element(element_name)
                        except KeyError:
                            msg = 'unable to add record element {ELEM} to section layout {SEC} - element {ELEM} is ' \
                                  'not defined in RecordElements'.format(ELEM=element_name, SEC=section)
                            logger.warning('RecordGroup {NAME}: {MSG}'.format(NAME=self.name, MSG=msg))

                            continue

                        if section_element.is_type('collection') or section_element.is_type('multiline'):
                            if row_len == 1:  # the long element is the only one defined for the row
                                row_elements.append(element_name)
                            else:  # long element defined on the same row as variable-length elements
                                remaining_elements.append(element_name)
                        else:  # record element is a variable length element
                            if column_num > 2:  # too many elements on the current row
                                msg = 'unable to add record element {ELEM} to row {ROW} of section layout {SEC} - ' \
                                      'only 3 elements are allowed per row'\
                                    .format(ELEM=element_name, SEC=section, ROW=row_num)
                                logger.warning('RecordGroup {NAME}: {MSG}'.format(NAME=self.name, MSG=msg))

                                remaining_elements.append(element_name)
                            else:
                                row_elements.append(element_name)
                                column_num += 1

                        used_section_elements.append(element_name)

                    section_layout[row_num] = row_elements
                    row_num += 1  # next row

                    for element_name in remaining_elements:
                        section_layout[row_num] = [element_name]
                        row_num += 1

            section_entry['Layout'] = section_layout
            self.sections[section] = section_entry

        for record_element in self.components:
            element_name = record_element.name
            if element_name not in used_section_elements:
                msg = 'no section provided for record element {ELEM}'.format(ELEM=element_name)
                logger.warning('RecordGroup {NAME}: {MSG}'.format(NAME=self.name, MSG=msg))
                #try:
                #    self.sections['_Default']['Elements'].append(element_name)
                #except KeyError:
                #    self.sections['_Default'] = {'Title': 'Record Information', 'Elements': [element_name]}

        for section in self.sections:
            if section not in used_sections:
                msg = 'no tab provided for section {SEC}'.format(SEC=section)
                logger.warning('RecordGroup {NAME}: {MSG}'.format(NAME=self.name, MSG=msg))
                #try:
                #    self.tabs['_Default']['Sections'].append(section)
                #except KeyError:
                #    self.tabs['_Default'] = {'Title': 'Details', 'Sections': [section]}
                continue

            section_bttn = '{}Bttn'.format(section)
            section_frame = '{}Frame'.format(section)
            section_width = '{}Width'.format(section)
            self.elements.update({i: '-{NAME}_{ID}_{ELEM}-'.format(NAME=self.name, ID=self.id, ELEM=i) for i in
                                  (section_bttn, section_frame, section_width)})
            self.bindings[self.key_lookup(section_bttn)] = section_bttn

        for i, tab_name in enumerate(self.tabs):
            tab_key = '{}Tab'.format(tab_name)
            tab_frame = '{}Col'.format(tab_name)
            self.elements.update({i: '-{NAME}_{ID}_{ELEM}-'.format(NAME=self.name, ID=self.id, ELEM=i) for i in
                                  (tab_key, tab_frame)})

        # Record report layout definition
        self.report = {}
        try:
            report = entry['Report']
        except KeyError:
            report = {}

        try:
            self.report['Title'] = report['Title']
        except KeyError:
            self.report['Title'] = None
        try:
            self.report['Subtitle'] = report['Subtitle']
        except KeyError:
            self.report['Subtitle'] = None
        try:
            self.report['ReportElements'] = report['ReportElements']
        except KeyError:
            self.report['ReportElements'] = {}
        try:
            self.report['Sections'] = report['Sections']
        except KeyError:
            self.report['Sections'] = {}

        self._minimum_height = 0
        self._padding = (0, 0)
        self._dimensions = (0, 0)
        self._display_elements = []

    def key_lookup(self, component, rev: bool = False):
        """
        Lookup a record element's GUI element key using the name of the component.

        Arguments:
            component (str): GUI component name (or key if rev is True) of the record element.

            rev (bool): reverse the element lookup map so that element keys are dictionary keys.
        """
        key_map = self.elements if rev is False else {j: i for i, j in self.elements.items()}
        try:
            key = key_map[component]
        except KeyError:
            msg = 'component {COMP} not found in list of record {NAME} components' \
                .format(COMP=component, NAME=self.name)
            logger.warning(msg)

            raise KeyError(msg)

        return key

    def bind_keys(self, window):
        """
        Add hotkey bindings to the record element.
        """
        for element in self.display_elements():
            element.bind_keys(window)

    def record_id(self, display: bool = False):
        """
        Convenience method for returning the record ID of the record object.
        """
        if display:
            return self._record_id.format_display()
        else:
            return self._record_id.data()

    def record_date(self, display: bool = False):
        """
        Convenience method for returning the record date of the record object.
        """
        if display:
            return self._record_date.format_display()
        else:
            return self._record_date.data()

    def initialize(self, data, new: bool = False, references: dict = None):
        """
        Initialize record attributes.

        Arguments:
            data (dict): dictionary, series, or dataframe containing the record data used to populate the data fields.

            new (bool): record is newly created [default: False]. New records allow editing of all data
                element fields, even those normally disabled.

            references (dict): optional dictionary of reference entry dataframes for association types configured for
                the parent record group. If provided, will not attempt to import references for the given association
                type but will use the reference entries provided instead.
        """
        # pd.set_option('display.max_columns', None)
        record_elements = self.components

        if not references:
            references = {}

        self.new = new
        record_entry = settings.records.fetch_rule(self.name)

        if isinstance(data, pd.Series):
            record_data = data.to_dict()
        elif isinstance(data, dict):
            record_data = data
        elif isinstance(data, pd.DataFrame):
            if data.shape[0] > 1:
                raise ImportError('more than one record provided to record of type "{TYPE}"'.format(TYPE=self.name))
            elif data.shape[0] < 1:
                raise ImportError('empty dataframe provided to record of type "{TYPE}"'.format(TYPE=self.name))
            else:
                record_data = data.iloc[0]
        else:
            raise AttributeError('unknown object type provided to record class "{TYPE}"'.format(TYPE=self.name))

        logger.info('RecordGroup {NAME}: initializing record'.format(NAME=self.name))
        logger.debug('RecordGroup {NAME}: {DATA}'.format(NAME=self.name, DATA=record_data))

        # Set header values from required columns
        try:
            self._record_id.format_value(record_data)
        except KeyError:
            raise ImportError('input record data is missing required column "{}"'.format(self.id_field))
        try:
            self._record_date.format_value(record_data)
        except KeyError:
            raise ImportError('input record data is missing required column "{}"'.format(self.date_field))

        if new:
            record_data[settings.creator_code] = user.uid
            record_data[settings.creation_date] = datetime.datetime.now()

        # Populate the record elements with data
        record_id = self.record_id()
        if not record_id:
            raise ImportError('failed to initialize the record - no record ID found in the data provided')

        logger.info('RecordGroup {NAME}: initialized record has ID {ID}'.format(NAME=self.name, ID=record_id))

        for record_element in record_elements:
            element_name = record_element.name
            if record_element.is_type('data_table') or record_element.is_type('data_list'):  # data table or data list
                element_fields = record_element.columns
                element_data = pd.Series(index=element_fields)
                for element_field in element_fields:
                    try:
                        element_data[element_field] = record_data[element_field]
                    except KeyError:
                        continue

                record_element.append(element_data)

            elif record_element.is_type('component'):  # component table
                assoc_rule = record_element.association_type
                ref_entry = settings.records.fetch_rule(record_element.record_type)

                logger.debug('RecordGroup {NAME}: initializing component table "{PARAM}" with association "{TYPE}"'
                             .format(NAME=self.name, PARAM=element_name, TYPE=assoc_rule))

                # Update the component table with parent data
                record_element.update_parent_data(record_data)

                # Attempt to add new components provided in the record data
                if element_name in record_data:  # new components provided
                    new_data = record_data[element_name]
                    if isinstance(new_data, pd.DataFrame) and not new_data.empty:
                        if record_element.id_column not in new_data.columns:  # IDs not yet created for the components
                            # Create IDs and default values for the new data
                            #new_data = self.create_components(record_element, record_data=new_data)
                            new_data = record_element.create_record(data=new_data)

                        record_element.append(new_data, new=True)
                    else:
                        logger.warning('RecordGroup {NAME}: record element {ELEM} can only accept input in the form '
                                       'of a dataframe'.format(NAME=self.name, ELEM=element_name))

                # Attempt to find existing components in the database

                # Load the reference entries defined by the given association rule
                if assoc_rule in references:  # use provided reference entries instead of importing from database
                    assoc_df = references[assoc_rule]
                    ref_data = assoc_df[(assoc_df['RecordID'] == record_id) & (~assoc_df['IsDeleted'])]
                else:
                    ref_data = record_entry.import_references(record_id, rule=assoc_rule)

                if ref_data.empty:
                    logger.debug('RecordGroup {NAME}: record {ID} has no current "{TYPE}" associations'
                                 .format(NAME=self.name, ID=record_id, TYPE=assoc_rule))
                    continue
                else:
                    n_ref = ref_data.shape[0]
                    logger.debug('RecordGroup {NAME}: found {N} associations of type "{TYPE}"'
                                 .format(NAME=self.name, N=n_ref, PARAM=element_name, TYPE=assoc_rule))

                import_ids = ref_data['ReferenceID']

                # Load any existing component records
                elem_data = ref_entry.load_records(import_ids, filters=record_element.import_filters)
                elem_data = elem_data[[i for i in elem_data.columns if i in record_element.columns]]

                record_element.append(elem_data, new=False)

            elif record_element.is_type('reference'):  # reference list or reference table
                assoc_rule = record_element.association_type

                logger.debug('RecordGroup {NAME}: initializing reference list "{PARAM}" with association "{TYPE}"'
                             .format(NAME=self.name, PARAM=element_name, TYPE=assoc_rule))

                # Load the reference entries defined by the given association rule
                if assoc_rule in references:  # use provided reference entries instead of importing from reference table
                    assoc_refs = references[assoc_rule]
                    ref_data = assoc_refs[(assoc_refs['RecordID'] == record_id) & (~assoc_refs['IsDeleted'])]
                else:
                    ref_data = record_entry.import_references(record_id, rule=assoc_rule)

                if ref_data.empty:
                    logger.debug('RecordGroup {NAME}: record {ID} has no current "{TYPE}" associations'
                                 .format(NAME=self.name, ID=record_id, TYPE=assoc_rule))
                    continue
                else:
                    n_ref = ref_data.shape[0]
                    logger.debug('RecordGroup {NAME}: found {N} associations of type "{TYPE}"'
                                 .format(NAME=self.name, N=n_ref, PARAM=element_name, TYPE=assoc_rule))

                record_element.append(ref_data, new=False)

            else:  # record variable element (dependent or record variables)
                try:
                    value = record_data[element_name]
                except KeyError:
                    logger.warning('RecordGroup {NAME}: input data is missing a value for data element "{PARAM}"'
                                   .format(NAME=self.name, PARAM=element_name))
                else:
                    if not pd.isna(value):
                        logger.debug('RecordGroup {NAME}: initializing data element "{PARAM}" with value "{VAL}"'
                                     .format(NAME=self.name, PARAM=element_name, VAL=value))
                        record_element.update_value(value)
                    else:
                        logger.debug('RecordGroup {NAME}: no value set for record element "{ELEM}"'
                                     .format(NAME=self.name, ELEM=element_name))
                        logger.debug('RecordGroup {NAME}: record element {ELEM} has default value "{VAL}"'
                                     .format(NAME=self.name, ELEM=element_name, VAL=record_element.data()))

        # Update any dependent record elements
        record_values = self.export_values(header=False).to_dict()
        try:
            dependent_elements = [i for i in record_elements if i.is_type('dependent')]
        except KeyError:
            pass
        else:
            for record_element in dependent_elements:
                try:
                    record_element.format_value(record_values)
                except Exception as e:
                    logger.warning('RecordGroup {NAME}: failed to set the value for dependent variable {ELEM} - {ERR}'
                                   .format(NAME=self.name, ELEM=record_element.name, ERR=e))

    def reset(self, window):
        """
        Reset record attributes.
        """
        self.new = False

        # Reset the header elements record ID and record date
        self._record_id.reset(window)
        self._record_date.reset(window)

        # Reset record data component elements
        for component in self.components:
            component.reset(window)

    def remove_unsaved_ids(self):
        """
        Remove any unsaved IDs associated with the record, including the records own ID.
        """
        record_id = self.record_id()
        record_elements = self.components

        record_entry = settings.records.fetch_rule(self.name)

        # Remove unsaved ID if record ID is found in the list of unsaved record IDs
        unsaved_ids = record_entry.get_unsaved_ids()
        if record_id in unsaved_ids:
            logger.debug('RecordType {NAME}: removing unsaved record ID {ID}'
                         .format(NAME=self.name, ID=record_id))
            record_entry.remove_unsaved_ids(record_ids=[record_id])

        # Remove unsaved components
        for record_element in record_elements:
            if not record_element.is_type('component'):
                continue

            comp_type = record_element.record_type
            if comp_type is None:
                continue
            else:
                comp_entry = settings.records.fetch_rule(comp_type)

            # Get a list of components added to the component table (i.e. not in the database yet)
            unsaved_ids = comp_entry.get_unsaved_ids()

            ids_to_remove = []
            comp_ids = record_element.row_ids(deleted=True)
            for row_id in comp_ids:
                if row_id in unsaved_ids:  # don't attempt to remove IDs if already in the database
                    ids_to_remove.append(row_id)

            logger.debug('RecordGroup {NAME}: removing unsaved component {COMP} IDs {IDS}'
                         .format(NAME=self.name, COMP=record_element.name, IDS=ids_to_remove))
            comp_entry.remove_unsaved_ids(record_ids=ids_to_remove)

    def fetch_element(self, identifier, by_key: bool = False, by_type: bool = False):
        """
        Fetch a record data element by name or event key.

        Arguments:
            identifier (str): identifying key.

            by_key (bool): fetch a record element by its element key [Default: False].

            by_type (bool): fetch record elements by element type [Default: False].
        """
        elements = self.record_elements()
        elements.extend(self.blanks)

        if by_key is True:
            # Remove any binding strings and get last component of element key
            match = re.match(r'-(.*?)-', identifier)
            if not match:
                raise KeyError('unknown format provided for element identifier {ELEM}'.format(ELEM=identifier))
            identifier = match.group(0)  # identifier returned if match
            element_key = match.group(1)  # element key part of the identifier after removing any binding

            element_type = element_key.split('_')[-1]
            element_names = []
            for element in elements:
                try:
                    element_name = element.key_lookup(element_type)
                except KeyError:
                    element_name = None

                element_names.append(element_name)
            logger.debug('RecordType {NAME}: searching record elements by element key {KEY}'
                         .format(NAME=self.name, KEY=identifier))
        elif by_type is True:
            logger.debug('RecordType {NAME}: searching record elements by element type {KEY}'
                         .format(NAME=self.name, KEY=identifier))
            element_names = [i.etype for i in elements]
        else:
            logger.debug('RecordType {NAME}: searching record elements by name {KEY}'
                         .format(NAME=self.name, KEY=identifier))
            element_names = [i.name for i in elements]

        element = [elements[i] for i, j in enumerate(element_names) if j == identifier]
        if len(element) == 0:
            raise KeyError('element {ELEM} not found in list of record {NAME} elements'
                           .format(ELEM=identifier, NAME=self.name))
        elif len(element) == 1 and by_type is False:
            return element[0]
        else:
            return element

    def run_event(self, window, event, values):
        """
        Perform a record action.
        """
        triggers = {'ValueEvent': False, 'ResizeEvent': False, 'DisplayEvent': False}

        # Collapse or expand a section frame
        if event in self.bindings:
            section_names = list(self.sections.keys())
            section_bttns = [self.key_lookup('{}Bttn'.format(i)) for i in section_names]
            if event in section_bttns:
                section_index = section_bttns.index(event)
                self.collapse_expand(window, section_names[section_index])

            return False

        # Record element events
        elem_events = self.record_events(components_only=True)
        record_elements = self.components

        # Get record element that is currently in focus
        try:
            focus_element = window.find_element_with_focus().Key
        except AttributeError:
            focus_element = None
        logger.debug('RecordType {NAME}: element in focus is {ELEM}'.format(NAME=self.name, ELEM=focus_element))

        # Check if any data elements are in edit mode
        for record_element in record_elements:
            try:
                edit_mode = record_element.edit_mode
            except AttributeError:  # record element is not an input field
                continue
            else:
                if not edit_mode:  # element is not currently being edited
                    continue

            # if focus_element not in record_element.elements:  # element is edited but is no longer in focus
            if focus_element not in record_element.bindings:  # element is edited but is no longer in focus
                logger.debug('RecordType {NAME}, Record {ID}: data element {PARAM} is no longer in focus - saving '
                             'changes'.format(NAME=self.name, ID=self.record_id(), PARAM=record_element.name))

                # Attempt to save the data element value
                record_element.run_event(window, 'Save', values)
            else:  # edited element is still in focus
                break

        # Run a record element event
        if event in elem_events:
            try:
                record_element = self.fetch_element(event, by_key=True)
            except KeyError as e:
                logger.error('Record {ID}: unable to find the record element associated with event key {KEY} - {ERR}'
                             .format(ID=self.record_id(), KEY=event, ERR=e))

                return False

            logger.debug('RecordType {NAME}: record event {EVENT} is in record element {ELEM}'
                         .format(NAME=self.name, EVENT=event, ELEM=record_element.name))
            triggers = record_element.run_event(window, event, values)

        if triggers['DisplayEvent']:
            self.update_display(window)

        if triggers['ResizeEvent']:
            window.visibility_changed()
            for section_name in self.sections:
                section_key = self.key_lookup('{}Col'.format(section_name))
                window[section_key].contents_changed()

        if triggers['ValueEvent']:
            # Update any element references with new values
            record_values = self.export_values(header=False)
            try:
                #element_references = self.fetch_element('dependent', by_type=True)
                element_references = [i for i in record_elements if i.is_type('dependent')]
            except KeyError:
                pass
            else:
                for record_element in element_references:
                    record_element.run_event(window, 'Update', record_values.to_dict())

        return True

    def update_display(self, window, size: tuple = None):
        """
        Update the record display.
        """
        record_id = self.record_id()
        display_elements = self.display_elements()

        # Update any dependent record elements first
        record_values = self.export_values(header=False)
        try:
            element_references = self.fetch_element('dependent', by_type=True)
        except KeyError:
            pass
        else:
            for record_element in element_references:
                record_element.format_value(record_values.to_dict())

        # Update the records header
        logger.debug('Record {ID}: updating the display header'.format(ID=record_id))
        self._record_id.update_display(window)
        self._record_date.update_display(window)

        # Update the record elements
        logger.debug('Record {ID}: updating record element displays'.format(ID=record_id))
        for record_element in display_elements:
            record_element.update_display(window)

        self.resize(window, size=size)

    def record_events(self, components_only: bool = False):
        """
        Return a list of available record event bindings.

        Arguments:
            components_only (bool): only retrieve event bindings for the record elements [Default: False].
        """
        events = {}
        if not components_only:
            events.update(self.bindings)

        # Add record component event bindings
        for record_element in self.components:
            events.update(record_element.bindings)

        return events

    def record_elements(self):
        """
        Return a list of all record elements.
        """
        elements = []
        elements.extend(self.components)

        return elements

    def display_elements(self):
        """
        Return a list of all display elements.
        """
        display_elements = []
        for element_name in self._display_elements:
            element = self.fetch_element(element_name)

            display_elements.append(element)

        return display_elements

    def check_required_parameters(self):
        """
        Verify that required record element values exist.
        """
        record_id = self.record_id()

        all_passed = True
        for element in self.components:
            passed = element.check_requirements()
            if not passed:
                msg = 'values missing for required element {ELEM}'.format(ELEM=element.description)
                logger.warning('Record {ID}: {MSG}'.format(ID=record_id, MSG=msg))
                mod_win2.popup_error(msg)

                all_passed = False
                break

        return all_passed

    def create_components(self, element, record_data=None):
        """
        Create new component records.
        """
        record_type = element.record_type

        if not element.is_type('component'):
            msg = 'failed to create component records for record element of type {TYPE} - component records can only ' \
                  'be created for record elements of type "component"'.format(TYPE=record_type)
            raise AttributeError(msg)

        header = element.columns
        record_date = self.record_date()

        if record_type is None:
            msg = 'required attribute "RecordType" missing from the configuration'
            raise AttributeError(msg)

        record_entry = settings.records.fetch_rule(record_type)

        # Create a new record object
        if record_data is None:
            record_data = pd.Series(index=header)
            creation_date = record_date
        else:
            if isinstance(record_data, pd.DataFrame):
                if record_data.empty:
                    msg = 'Record {ID}: unable to create record IDs when no components were provided' \
                        .format(ID=self.record_id())
                    logger.warning(msg)

                    return record_data
                creation_date = [record_date for _ in range(record_data.shape[0])]
            elif isinstance(record_data, pd.Series):  # single row (pandas series)
                creation_date = record_date
            elif isinstance(record_data, dict):
                record_data = pd.DataFrame(record_data)
                creation_date = [record_date for _ in range(record_data.shape[0])]
            else:
                record_data = pd.Series(index=header)
                creation_date = record_date

        record_id = record_entry.create_record_ids(creation_date, offset=settings.get_date_offset())
        if not record_id:
            msg = 'unable to create an ID for the new record'
            raise AssertionError(msg)

        record_data['RecordID'] = record_id
        record_data['RecordDate'] = creation_date
        record_data[settings.creator_code] = user.uid
        record_data[settings.creation_date] = datetime.datetime.now()

        # Use record data to populate component fields when fields are shared between the component and record
        if isinstance(record_data, pd.Series):
            record_data = record_data.to_frame().T

        defaults = self.export_values(header=False).to_dict()
        for default_col in defaults:
            if default_col in header:
                default_value = defaults[default_col]
                if pd.isna(default_value):
                    continue

                try:
                    record_data[default_col].fillna(default_value, inplace=True)
                except KeyError:
                    record_data[default_col] = default_value

        return record_data.squeeze()

    def export_values(self, header: bool = True, edited_only: bool = False):
        """
        Export record data as a Series.

        Arguments:
            header (bool): include header components in the exported values [Default: True].

            edited_only (bool): only include values for record elements that were edited [Default: False].

        Returns:
            values (Series): values of the record's elements.
        """
        values = {}

        if header:
            record_id = self._record_id
            record_date = self._record_date

            # Add header values
            values[record_id.name] = record_id.data()
            values[record_date.name] = record_date.data()

        # Add parameter values
        record_elements = self.components
        for record_element in record_elements:
            elem_values = record_element.export_values(edited_only=edited_only)
            values = {**values, **elem_values}

        return pd.Series(values)

    def export_associations(self, association_rule: str = None):
        """
        Export record data as a Series.

        Arguments:
            association_rule (str): only export associated records under the given association rule
                [Default: export all].

        Returns:
            df (DataFrame): reference dataframe of associated records.
        """
        ref_df = pd.DataFrame(columns=['RecordID', 'ReferenceID', 'ReferenceDate', 'RecordType', 'ReferenceType',
                                       'ReferenceNotes', 'ReferenceWarnings', 'IsChild', 'IsHardLink', 'IsApproved',
                                       'IsDeleted'])

        record_elements = self.record_elements()

        #try:
        #    refbox_elements = self.fetch_element('reference', by_type=True)
        #except KeyError:
        #    refbox_elements = []
        ref_elements = [i for i in record_elements if i.is_type('reference')]
        for ref_element in ref_elements:
            ref_rule = ref_element.association_type
            if association_rule and (association_rule != ref_rule):
                continue

            data = ref_element.data()
            ref_df = ref_df.append(data, ignore_index=True)

        #try:
        #    comp_elements = self.fetch_element('components', by_type=True)
        #except KeyError:
        #    comp_elements = []
        comp_elements = [i for i in record_elements if i.is_type('component')]
        for comp_tbl in comp_elements:
            comp_rule = comp_tbl.association_type
            if association_rule and association_rule != comp_rule:
                continue

            #data = comp_tbl.export_references(self.record_id())
            data = comp_tbl.export_references()
            ref_df = ref_df.append(data, ignore_index=True)

        return ref_df

    def prepare_delete_statements(self, statements: dict = None):
        """
        Prepare statements for deleting the record and child records from the database.

        Arguments:
            statements (dict): optional dictionary of transaction statements to add the delete statements to.

        Returns:
            statements (dict): dictionary of transaction statements.
        """
        record_id = self.record_id()
        record_entry = settings.records.fetch_rule(self.name)
        association_rules = record_entry.association_rules

        if not statements:
            statements = {}

        nchild = 0
        nlink = 0
        for rule_name in association_rules:
            rule = association_rules[rule_name]
            assoc_type = rule['AssociationType']

            if assoc_type == 'child':
                continue

            ref_df = record_entry.import_references(record_id, rule=rule_name)

            # Subset references to include those that are child records or hard-linked
            if assoc_type == 'parent':  # reference table may contain child records
                nrow = ref_df[ref_df['IsChild']].shape[0]
                nchild += nrow
            elif assoc_type == 'reference':  # reference table may contain hard-linked records
                nrow = ref_df[ref_df['IsHardLink']].shape[0]
                nlink += nrow
            else:  # child records don't affect parent records
                continue

        if nlink > 0 or nchild > 0:  # Record is hard-linked to other records
            msg = 'Deleting record {ID} will also delete {N} dependent records and {NH} direct hard-linked records ' \
                  'as well. Would you like to continue with record deletion?'.format(ID=record_id, N=nchild, NH=nlink)
        else:
            msg = 'Are you sure that you would like to delete this record?'

        user_input = mod_win2.popup_confirm(msg)
        if user_input != 'OK':
            raise IOError('user selected to cancel record deletion')

        # Prepare statements for the removal of the record
        logger.debug('Record {ID}: preparing database transaction statements'.format(ID=record_id))

        try:
            statements = record_entry.delete_database_records(record_id, statements=statements)
        except Exception as e:
            msg = 'failed to delete record from the database - {ERR}'.format(ERR=e)
            logger.exception('Record {ID}: {MSG}'.format(ID=record_id, MSG=msg))

            raise

        return statements

    def delete(self, statements: dict = None):
        """
        Delete the record and child records from the database.

        Arguments:
            statements (dict): optional dictionary of transaction statement dictionary to add to.

        Returns:
            success (bool): record delete was successful.
        """
        record_id = self.record_id()

        # Prepare deletion statements for the record and any child and hard-linked records
        try:
            statements = self.prepare_delete_statements(statements=statements)
        except Exception as e:
            msg = 'Record {ID}: failed to prepare database transaction statements for deletion - {ERR}' \
                .format(ID=record_id, ERR=e)
            logger.exception(msg)
            mod_win2.popup_error(msg)

            return False
        else:
            if len(statements) < 1:
                logger.debug('Record {ID}: no records needed deleting from the database'.format(ID=record_id))

                return True

        # Write record to the database
        logger.info('preparing to delete record {ID} and any child records'.format(ID=record_id))
        sstrings = []
        psets = []
        for i, j in statements.items():
            sstrings.append(i)
            psets.append(j)

        success = user.write_db(sstrings, psets)
        # success = True
        print(statements)

        return success

    def prepare_save_statements(self, statements: dict = None, save_all: bool = False):
        """
        Prepare to the statements for saving the record to the database.

        Arguments:
            statements (dict): optional dictionary of transaction statement dictionary to add to.

            save_all (bool): save all record element values, regardless of whether they were edited or not
                [Default: False]

        Returns:
            statements (dict): dictionary of transaction statements.
        """
        if not statements:
            statements = {}

        record_id = self.record_id()
        record_entry = settings.records.fetch_rule(self.name)
        record_elements = self.record_elements()

        # Verify that required parameters have values
        can_continue = self.check_required_parameters()
        if not can_continue:
            msg = 'failed to prepare save statements - not all required parameters have values'
            logger.warning('Record {ID}: {MSG}'.format(ID=record_id, MSG=msg))

            raise AttributeError(msg)

        # Prepare to save the record values
        logger.debug('Record {ID}: preparing database transaction statements'.format(ID=record_id))
        try:
            logger.debug('RecordGroup {NAME}, Record {ID}: exporting record values'
                         .format(NAME=self.name, ID=record_id))
            record_data = self.export_values()
            statements = record_entry.save_database_records(record_data, id_field=self.id_field, statements=statements)
        except Exception as e:
            msg = 'failed to save record "{ID}" - {ERR}'.format(ID=record_id, ERR=e)
            logger.exception(msg)

            raise
        else:
            del record_data

        # Prepare to save record references
        #try:
        #    ref_elements = self.fetch_element('reference', by_type=True)
        #except KeyError:
        #    ref_elements = []
        ref_elements = [i for i in record_elements if i.is_type('reference')]
        for ref_element in ref_elements:
            element_name = ref_element.name
            print('Saving references from record element {}'.format(element_name))
            association_rule = ref_element.association_type

            if self.new or save_all:  # export all reference entries if record is new or if indicated to do so
                logger.debug('Record {ID}: preparing export statements for all "{ASSOC}" references in {REF}'
                             .format(ID=record_id, ASSOC=association_rule, REF=element_name))
                ref_data = ref_element.data()
            else:  # export only the added or edited reference entries
                logger.debug('Record {ID}: preparing export statements for edited or new "{ASSOC}" references in {REF}'
                             .format(ID=record_id, ASSOC=association_rule, REF=element_name))
                ref_data = ref_element.data(edited_rows=True, added_rows=True)  # edited or added
            statements = record_entry.save_database_references(ref_data, association_rule, statements=statements)

            logger.debug('Record {ID}: preparing export statements for deleted "{ASSOC}" references in {REF}'
                         .format(ID=record_id, ASSOC=association_rule, REF=element_name))
            deleted_df = ref_element.data(deleted_rows=True, added_rows=False)  # existing deleted only
            if not deleted_df.empty:
                statements = record_entry.delete_database_references(deleted_df, association_rule, statements=statements)

        # Prepare to save record components
        #try:
        #    component_tables = self.fetch_element('component', by_type=True)
        #except KeyError:
        #    component_tables = []
        component_tables = [i for i in record_elements if i.is_type('component')]
        for comp_table in component_tables:
            element_name = comp_table.name
            comp_type = comp_table.record_type

            comp_entry = settings.records.fetch_rule(comp_type)

            if self.new or save_all:
                logger.debug('Record {ID}: preparing export statements for all components in component table "{COMP}"'
                             .format(ID=record_id, COMP=element_name))
                exist_df = comp_table.data()
                #ref_data = comp_table.export_references(record_id)
                ref_data = comp_table.export_references()
            else:
                logger.debug('Record {ID}: preparing export statements for only the edited components in component '
                             'table {COMP}'.format(ID=record_id, COMP=element_name))
                #ref_data = comp_table.export_references(record_id, edited_only=True)
                ref_data = comp_table.export_references(edited_only=True)
                exist_df = comp_table.data(edited_rows=True, added_rows=True)

            deleted_df = comp_table.data(deleted_rows=True, added_rows=False)  # existing deleted only

            if deleted_df.empty and exist_df.empty:
                logger.debug('Record {ID}: no components in component table {COMP} available to export'
                             .format(ID=record_id, COMP=element_name))
                continue

            logger.debug('Record {ID}: preparing statements for component table "{TBL}"'
                         .format(ID=record_id, TBL=element_name))

            assoc_rule_name = comp_table.association_type
            assoc_rule = comp_entry.association_rules.get(assoc_rule_name)
            if assoc_rule['AssociationType'] == 'child':  # component records are a child records
                # Fully remove deleted component records if parent-child relationship
                deleted_ids = comp_table.row_ids(indices=deleted_df.index)
                if not deleted_df.empty:
                    statements = comp_entry.delete_database_records(deleted_ids, statements=statements)

                    # Subset ref_data so that references are not updated twice
                    ref_data = ref_data[~ref_data['ReferenceID'].isin(deleted_ids)]

                # Set reference flags
                ref_data['IsChild'] = True
            else:
                # Set reference flags
                ref_data['IsChild'] = False

            statements = record_entry.save_database_references(ref_data, assoc_rule_name, statements=statements)

            # Prepare transaction statements for the component records
            try:
                statements = comp_entry.save_database_records(exist_df, statements=statements)
            except Exception as e:
                msg = 'failed to save record "{ID}" - {ERR}'.format(ID=record_id, ERR=e)
                logger.error(msg)

                raise

        return statements

    def save(self, statements: dict = None, save_all: bool = False):
        """
        Save the record and child records to the database.

        Arguments:
            statements (dict): optional dictionary of transaction statement dictionary to add to.

            save_all (bool): save all record element values, regardless of whether they were edited or not
                [Default: False]

        Returns:
            success (bool): record save was successful.
        """
        record_id = self.record_id()

        try:
            statements = self.prepare_save_statements(statements=statements, save_all=save_all)
        except Exception as e:
            msg = 'failed to save record - {ERR}'.format(ERR=e)
            logger.exception('RecordType {NAME}: Record {ID}: {MSG}'.format(NAME=self.name, ID=record_id, MSG=msg))

            return False
        else:
            if len(statements) < 1:
                logger.error('RecordType {NAME}: Record {ID}: failed to save record - failed to prepare transaction '
                             'statements'.format(NAME=self.name, ID=record_id))

                return False

        logger.info('RecordType {NAME}: Record {ID}: preparing to save record and record components'
                    ''.format(NAME=self.name, ID=record_id))
        sstrings = []
        psets = []
        for i, j in statements.items():
            sstrings.append(i)
            psets.append(j)

        success = user.write_db(sstrings, psets)
        #success = True
        print('final save statements:')
        print(statements)

        return success

    def _format_value_str(self, text_str):
        """
        Format a text string that incorporates record element values.
        """
        record_id = self._record_id.format_display()
        record_date = self._record_date.format_display()
        elements = self.components

        # Update the report title with record element values when part of the title
        try:
            value_components = re.findall(r'{(.*?)}', text_str)
        except TypeError:
            value_components = []
        else:
            logger.debug('RecordType {NAME}: string value components are {COMPS}'
                         .format(NAME=self.name, COMPS=value_components))

        value_params = {}
        if 'RecordID' in value_components:
            value_params['RecordID'] = record_id
        if 'RecordDate' in value_components:
            value_params['RecordDate'] = record_date

        for element in elements:
            element_name = element.name

            # Check if parameter composes part of title
            if element_name in value_components:
                display_value = element.format_display()
                logger.debug('RecordType {NAME}: adding record element value "{VAL}" to the text string'
                             .format(NAME=self.name, VAL=display_value))
                value_params[element_name] = display_value

        try:
            formatted_text = text_str.format(**value_params)
        except KeyError as e:
            logger.error('RecordType {NAME}: failed to format text string - {ERR}'.format(NAME=self.name, ERR=e))
            formatted_text = text_str

        logger.info('RecordType {NAME}: formatted text string is {TITLE}'
                    .format(NAME=self.name, TITLE=formatted_text))

        return formatted_text

    def generate_report(self):
        """
        Generate a summary report for the record.
        """
        report_def = self.report
        record_type = self.name

        logger.info('RecordType {NAME}: generating the template for the record report'.format(NAME=record_type))

        # Generate the report title
        title_str = report_def['Title']
        if title_str is None:
            record_id = self._record_id.format_display()
            record_date = self._record_date.format_display()
            report_title = '{TITLE}: {ID} ({DATE})'.format(TITLE=self.title, ID=record_id, DATE=record_date)
        else:
            report_title = self._format_value_str(str(title_str))

        subtitle_str = report_def['Subtitle']
        if subtitle_str is None:
            report_subtitle = ''
        else:
            report_subtitle = self._format_value_str(str(subtitle_str))

        # Create the body of the report
        element_settings = report_def['ReportElements']

        sections = report_def['Sections']
        report_body = {}
        for section in sections:
            section_entry = sections[section]
            section_template = {}

            try:
                section_template['title'] = section_entry['Title']
            except KeyError:
                section_template['title'] = section

            try:
                section_template['level'] = int(section_entry['Level'])
            except (KeyError, ValueError):
                section_template['level'] = 0

            try:
                section_elements = section_entry['Elements']
            except KeyError:
                logger.warning('RecordType {NAME}: no elements defined under report section {SEC}'
                               .format(NAME=record_type, SEC=section))
                section_elements = []

            elements = {}
            for report_element in section_elements:
                try:
                    element_entry = element_settings[report_element]
                except KeyError:
                    element_entry = {}

                try:
                    element_name = element_entry['RecordElement']
                except KeyError:
                    element_name = report_element

                try:
                    element = self.fetch_element(element_name)
                except KeyError:
                    logger.warning('RecordType {NAME}, Section {SEC}: unknown record element "{COMP}" defined in the '
                                   'report configuration'.format(NAME=record_type, SEC=section, COMP=element_name))
                    continue

                try:
                    elem_title = element_entry['Title']
                except KeyError:
                    elem_title = element.description

                if element.is_type('table'):  # record element is a table
                    # Subset table rows based on configured subset rules
                    try:
                        sub_rule = element_entry['Subset']
                    except KeyError:
                        subset_df = element.data()
                    else:
                        try:
                            subset_df = element.subset(sub_rule)
                        except ValueError as e:
                            logger.warning('RecordType {NAME}, Section {SEC}: failed to subset record element {ELEM} '
                                           'on rule "{SUB}" - {ERR}'
                                           .format(NAME=record_type, ELEM=element_name, SEC=section, SUB=sub_rule, ERR=e))
                            subset_df = element.data()

                    # Select columns to display in the report
                    try:
                        subset_df = subset_df[element_entry['DisplayColumns']]
                    except KeyError:
                        subset_df = subset_df[list(element.display_columns)]

                    if subset_df.empty:
                        logger.warning('RecordType {NAME}, Section {SEC}: no entries remaining in record element {ELEM}'
                                       .format(NAME=record_type, ELEM=element_name, SEC=section))
                        html_out = '<p>N/A</p>'
                    else:
                        # Format table for display
                        display_df = subset_df.copy()
                        for column in subset_df.columns:
                            try:
                                display_df[column] = element.format_display_column(subset_df, column)
                            except Exception as e:
                                msg = 'failed to format display column "{COL}" - {ERR}'.format(COL=column, ERR=e)
                                logger.exception('RecordType {NAME}, Section {SEC}: {MSG}'
                                                 .format(NAME=record_type, SEC=section, MSG=msg))

                        # Index rows using grouping list in configuration
                        try:
                            grouping = element_entry['Group']
                        except KeyError:
                            grouped_df = display_df
                        else:
                            grouped_df = display_df.set_index(grouping).sort_index()

                        html_str = grouped_df.to_html(header=False, index_names=False, float_format='{:,.2f}'.format,
                                                      sparsify=True, na_rep='')

                        # Highlight errors in html string
                        annotations = element.annotate_rows(grouped_df.reset_index())
                        colors = {i: element.annotation_rules[j]['BackgroundColor'] for i, j in annotations.items()}
                        try:  # colors should be a dictionary of row index with matching color
                            html_out = replace_nth(html_str, '<tr>', '<tr style="background-color: {}">', colors)
                        except Exception as e:
                            logger.warning('RecordType {NAME}, Section {SEC}: failed to annotate output - {ERR}'
                                           .format(NAME=record_type, SEC=section, ERR=e))
                            html_out = html_str

                        # Add summary totals
                        try:
                            total_col = element_entry['Totals']
                        except KeyError:
                            pass
                        else:
                            if total_col in subset_df.columns:
                                try:
                                    col_total = element.summarize_column(total_col, indices=subset_df.index.tolist())
                                except Exception as e:
                                    msg = 'failed to summarize column "{COL}" - {ERR}'.format(COL=total_col, ERR=e)
                                    logger.warning('RecordType {NAME}, Section {SEC}: {MSG}'
                                                   .format(NAME=record_type, SEC=section, MSG=msg))
                                else:
                                    if isinstance(col_total, float):
                                        summ_value = '{:,.2f}'.format(col_total)
                                    else:
                                        summ_value = '{}'.format(col_total)

                                    soup = BeautifulSoup(html_out, 'html.parser')

                                    # Add a totals footer to the table
                                    footer = soup.new_tag('tfoot')
                                    footer_row = soup.new_tag('tr')
                                    footer_header = soup.new_tag('td')
                                    footer_header['id'] = 'total'
                                    footer_header['colspan'] = '{}'.format(subset_df.shape[1] - 1)
                                    footer_header['style'] = 'text-align:right; font-weight:bold;'
                                    footer_header.string = 'Total:'
                                    footer_row.append(footer_header)

                                    footer_data = soup.new_tag('td')
                                    footer_data.string = summ_value
                                    footer_row.append(footer_data)

                                    footer.append(footer_row)
                                    soup.table.append(footer)

                                    html_out = soup.decode()
                            else:
                                msg = 'Totals column "{COL}" not found in list of output columns'.format(COL=total_col)
                                logger.warning('RecordType {NAME}, Section {SEC}: {MSG}'
                                               .format(NAME=report_title, SEC=section, MSG=msg))

                    elem_value = html_out
                    element_template = {'type': 'table', 'title': elem_title, 'value': elem_value}
                else:
                    elem_value = element.format_display()
                    if not elem_value or elem_value == "":
                        elem_value = 'N/A'

                    element_template = {'type': 'variable', 'title': elem_title, 'value': elem_value}

                elements[report_element] = element_template

            section_template['elements'] = elements

            report_body[section] = section_template

        template = {'title': report_title, 'subtitle': report_subtitle, 'body': report_body}

        return template

    def save_report(self, filename):
        """
        Generate the audit record report and save the report to the output file.

        Arguments:
            filename (str): save record report to the provided output file.

        Returns:
            success (bool): saving report was successful.
        """
        css_url = settings.report_css

        logger.info('AuditRule {NAME}: saving audit record report to {FILE}'
                    .format(NAME=self.name, FILE=filename))

        try:
            template_vars = self.generate_report()
        except Exception as e:
            msg = 'failed to generate the record report - {ERR}'.format(ERR=e)
            logger.exception('AuditRule {NAME}: {MSG}'.format(NAME=self.name, MSG=msg))

            return False

        env = Environment(loader=FileSystemLoader(os.path.dirname(os.path.abspath(settings.report_template))))
        template = env.get_template(os.path.basename(os.path.abspath(settings.report_template)))
        html_out = template.render(template_vars)
        path_wkhtmltopdf = settings.wkhtmltopdf
        config = pdfkit.configuration(wkhtmltopdf=path_wkhtmltopdf)
        try:
            pdfkit.from_string(html_out, filename, configuration=config, css=css_url,
                               options={'enable-local-file-access': None})
        except Exception as e:
            msg = 'failed to generate the record report - unable to write the PDF ({ERR})'.format(ERR=e)
            logger.error('AuditRule {NAME}: {MSG}'.format(NAME=self.name, MSG=msg))
            success = False
        else:
            success = True

        return success

    def layout(self, size, padding: tuple = None, view_only: bool = False):
        """
        Generate a GUI layout for the database record.

        Arguments:
            size (tuple): size (width, height) of the record layout.

            padding (tuple): padding around the record layout in pixels.

            view_only (bool): disallow editing of record components [Default: False].
        """
        width, height = size

        # Permissions
        level = self.level
        is_new = self.new

        logger.debug('RecordType {NAME}: creating {NEW}record layout at level {LEVEL}'
                     .format(NAME=self.name, NEW=('new ' if is_new else ''), LEVEL=level))

        editable = False if view_only is True else True

        # Element parameters
        bg_col = mod_const.DEFAULT_BG_COLOR
        text_col = mod_const.DEFAULT_TEXT_COLOR
        inactive_col = mod_const.DISABLED_BG_COLOR
        select_col = mod_const.SELECTED_TEXT_COLOR
        headings_bg_col = mod_const.PANELBAR_COLOR
        headings_text_col = mod_const.WHITE_TEXT_COLOR

        bold_font = mod_const.HEADING_FONT
        main_font = mod_const.MAIN_FONT

        pad_el = mod_const.ELEM_PAD
        pad_v = mod_const.VERT_PAD

        padding = self._padding if not padding else padding
        self._padding = padding
        pad_w, pad_h = padding

        scroll_w = mod_const.SCROLL_WIDTH
        bar_h = 32
        header_h = 50

        min_h = header_h + bar_h
        self._minimum_height = min_h

        tab_w = width - pad_w * 2
        tab_h = height - (header_h + bar_h + pad_h * 2)
        sec_w = tab_w - scroll_w

        # Layout elements

        # Record header
        headers = [sg.Canvas(size=(0, header_h), background_color=bg_col)]
        headers.append(self._record_id.layout(size=(20, 1), padding=((0, pad_h), 0)))
        headers.append(self._record_date.layout(size=(20, 1)))
        header_key = self.key_lookup('Header')
        header_layout = sg.Col([headers], key=header_key, expand_x=True, background_color=bg_col, justification='l',
                               vertical_alignment='c')

        # Create layout for the record tabs
        tabs = self.tabs
        sections = self.sections

        tab_group = []
        display_elements = []
        for tab_name in tabs:
            tab_entry = tabs[tab_name]

            tab_title = '{:^40}'.format(tab_entry.get('Title', tab_name))
            tab_icon = tab_entry.get('Icon', None)
            tab_icon_path = None if tab_icon is None else settings.get_icon_path(tab_icon)

            tab_sections = tab_entry['Sections']
            if len(tab_sections) < 1:
                continue

            # Create the layouts for the tab sections
            sections_layout = []
            for section_name in sections:
                if section_name not in tab_sections:  # section belongs to a different tab
                    continue

                section_entry = sections[section_name]

                # Section header
                section_title = section_entry['Title']
                section_bttn_key = self.key_lookup('{}Bttn'.format(section_name))
                header_left = [
                    sg.Text(section_title, text_color=headings_text_col, background_color=headings_bg_col,
                            font=bold_font)]
                header_right = [sg.Button('', image_data=mod_const.COLLAPSE_ICON, key=section_bttn_key, disabled=False,
                                          button_color=(headings_text_col, headings_bg_col), border_width=0,
                                          visible=True, metadata={'visible': True, 'disabled': False})]

                if 'HeadingElement' in section_entry:
                    heading_element_name = section_entry['HeadingElement']
                    heading_element = self.fetch_element(heading_element_name)
                    if heading_element.is_type('variable'):
                        heading_element.arrangement = 'h'  # force horizontal layout
                        heading_element.align = False  # disable forced alignment with other elements
                        heading_element.justification = 'r'  # force right justification of element components
                        heading_element_layout = heading_element.layout(padding=((0, pad_el * 2), 0), level=level,
                                                                        bg_color=headings_bg_col, editable=False)
                        header_right.insert(0, heading_element_layout)

                        display_elements.append(heading_element_name)
                    else:
                        logger.warning('RecordType {NAME}: record element {ELEM} with unsupported element type '
                                       '{TYPE} is attempting to be used as a header element'
                                       .format(NAME=self.name, ELEM=heading_element_name, TYPE=heading_element.etype))

                section_w_key = self.key_lookup('{}Width'.format(section_name))
                section_header = [sg.Col([[sg.Canvas(key=section_w_key, size=(sec_w, 0),
                                                     background_color=headings_bg_col)],
                                          [sg.Canvas(size=(0, bar_h), background_color=headings_bg_col),
                                           sg.Col([header_left], justification='l', expand_x=True,
                                                  background_color=headings_bg_col),
                                           sg.Col([header_right], justification='r', background_color=headings_bg_col)
                                           ]], background_color=headings_bg_col, pad=(0, (0, pad_el*2)), expand_x=True)]
                sections_layout.append(section_header)

                # Section body layout
                section_layout = []
                try:
                    section_elements = section_entry['Layout']
                except KeyError:
                    msg = 'section {} does not have a layout for its elements'.format(section_name)
                    logger.warning('RecordType {NAME}: {MSG}'.format(NAME=self.name, MSG=msg))

                    continue

                for section_row in section_elements:
                    row_elements = section_elements[section_row]
                    n_elem = len(row_elements)

                    row = []
                    for i, elem_name in enumerate(row_elements):
                        element = self.fetch_element(elem_name)

                        elem_pad = (0, (0, pad_v)) if element.is_type('collection') else (0, 0)
                        elem_layout = element.layout(editable=editable, overwrite=is_new, level=level, padding=elem_pad)

                        row.append(elem_layout)
                        if i != (n_elem - 1):  # don't add alignment element after the last element in the row
                            row.append(sg.Push(background_color=bg_col))

                        if not element.is_type('blank'):
                            display_elements.append(elem_name)

                    section_layout.append(row)

                section_panel_key = self.key_lookup('{}Frame'.format(section_name))
                sections_layout.append([sg.pin(sg.Col(section_layout, key=section_panel_key, pad=(0, (0, pad_v)),
                                                      background_color=bg_col, visible=True, expand_x=True,
                                                      metadata={'visible': True}),
                                               expand_x=True)])

            tab_key = self.key_lookup('{}Tab'.format(tab_name))
            tab_frame_key = self.key_lookup('{}Col'.format(tab_name))
            tab_layout = sg.Tab(tab_title, [[sg.Col(sections_layout, key=tab_frame_key, size=(tab_w, tab_h),
                                                    pad=(0, pad_v), background_color=bg_col,
                                                    scrollable=True, vertical_scroll_only=True, vertical_alignment='t',
                                                    element_justification='l', expand_x=True, expand_y=True)]],
                                key=tab_key, image_source=tab_icon_path, background_color=bg_col)
            tab_group.append(tab_layout)

        main_layout = sg.TabGroup([tab_group], key=self.key_lookup('TG'),
                                  background_color=bg_col, tab_background_color=inactive_col,
                                  selected_background_color=bg_col, selected_title_color=select_col,
                                  title_color=text_col, border_width=0, tab_location='topleft', font=main_font)

        # Pane elements must be columns
        record_key = self.key_lookup('Element')
        layout = [[sg.Col([[header_layout], [main_layout]], key=record_key, pad=padding,
                          background_color=bg_col, expand_x=True, expand_y=True)]]

        self._dimensions = size
        self._display_elements = display_elements

        return layout

    def resize(self, window, size: tuple = None):
        """
        Resize the record elements.
        """
        width, height = size if size is not None else self._dimensions

        logger.debug('Record {ID}: resizing display to {W}, {H}'.format(ID=self.record_id(), W=width, H=height))

        # Expand the record containers
        min_h = self._minimum_height
        record_h = height if height > min_h else min_h

        pad_w, pad_h = self._padding
        scroll_w = mod_const.SCROLL_WIDTH
        section_w_pad = pad_w * 2 + scroll_w  # padding around the section column, including column scrollbar
        spacer_w = mod_const.SPACER_WIDTH  # field element column spacer width
        tab_h = 22
        header_h = 32

        # Set the size of the record container
        record_w = width - pad_w * 2
        record_size = (record_w, record_h)
        mod_lo.set_size(window, self.key_lookup('Element'), record_size)

        # Set the size of the tab containers and sections
        bffr_h = tab_h + header_h + pad_h * 2
        tab_size = (record_w, height - bffr_h)
        for tab_name in self.tabs:
            tab_key = self.key_lookup('{}Col'.format(tab_name))
            mod_lo.set_size(window, tab_key, tab_size)
            window[tab_key].expand(expand_x=True)

        for section_name in self.sections:
            section_w_key = self.key_lookup('{}Width'.format(section_name))
            window[section_w_key].set_size(size=(record_w - scroll_w, None))

        # Set the size of the header elements
        for header in (self._record_id, self._record_date):
            header.resize(window)

        # Resize record elements
        for section_name in self.sections:
            section_entry = self.sections[section_name]

            # Resize any section header elements
            if 'HeadingElement' in section_entry:
                heading_element_name = section_entry['HeadingElement']
                heading_element = self.fetch_element(heading_element_name)
                heading_element.resize(window)

            # Resize section body elements
            elem_h = None
            column_div = divmod(width - section_w_pad - spacer_w * 2, 3)
            column_w = column_div[0] + column_div[1]

            section_layout = section_entry['Layout']
            for row in section_layout:
                row_elements = section_layout[row]
                n_col = len(row_elements)

                for i, element_name in enumerate(row_elements):
                    record_element = self.fetch_element(element_name)

                    if n_col == 1:  # row contains 1 columns spanning the entire row
                        elem_w = width - section_w_pad
                    elif n_col == 2:
                        if i == 0:  # first element in the row of two elements takes up two spaces
                            elem_w = column_w * n_col + spacer_w * (n_col - 1)
                        else:  # last element in the row of two elements takes up only one space
                            elem_w = column_w
                    else:  # row contains 3 columns of equal size
                        elem_w = column_w

                    elem_size = (elem_w, elem_h)
                    record_element.resize(window, size=elem_size)

        self._dimensions = (width, height)

    def collapse_expand(self, window, section):
        """
        Collapse record frames.
        """
        hide_key = self.key_lookup('{}Bttn'.format(section))
        frame_key = self.key_lookup('{}Frame'.format(section))

        if window[frame_key].metadata['visible'] is True:  # already visible, so want to collapse the frame
            logger.debug('RecordType {NAME}, Record {ID}: collapsing section {FRAME}'
                         .format(NAME=self.name, ID=self.record_id(), FRAME=section))
            window[hide_key].update(image_data=mod_const.EXPAND_ICON)
            window[frame_key].update(visible=False)

            window[frame_key].metadata['visible'] = False
        else:  # not visible yet, so want to expand the frame
            logger.debug('RecordType {NAME}, Record {ID}: expanding section {FRAME}'
                         .format(NAME=self.name, ID=self.record_id(), FRAME=section))
            window[hide_key].update(image_data=mod_const.COLLAPSE_ICON)
            window[frame_key].update(visible=True)

            window[frame_key].metadata['visible'] = True

def replace_nth(s, sub, new, ns):
    """
    Replace the nth occurrence of an substring in a string
    """
    if isinstance(ns, str):
        ns = [ns]

    where = [m.start() for m in re.finditer(sub, s)]
    new_s = s
    for count, start_index in enumerate(where):
        if count not in ns:
            continue

        if isinstance(ns, dict):
            new_fmt = new.format(ns[count])
        else:
            new_fmt = new

        before = new_s[:start_index]
        after = new_s[start_index:]
        after = after.replace(sub, new_fmt, 1)  # only replace first instance of substring
        new_s = before + after

    return new_s

def initialize_element(parent, name, entry):
    """
    Initialize a record element based on the element's configured element type.
    """
    try:
        etype = entry['ElementType']
    except KeyError:
        raise AttributeError('element entry is missing the required field "ElementType"'.format(NAME=name))

    # Set the object type of the record element.
    if etype in ('data_table', 'table'):
        element_class = mod_elem.DataTable
    elif etype in ('component_table', 'components'):
        element_class = mod_elem.ComponentTable
    elif etype in ('association_table',):
        element_class = mod_elem.AssociationTable
    elif etype in ('reference_table',):
        element_class = mod_elem.ReferenceTable
    elif etype in ('data_list', 'list'):
        element_class = mod_elem.DataList
    elif etype in ('reference_list', 'reference'):
        element_class = mod_elem.ReferenceList
    elif etype in ('dependent_variable', 'dependent'):
        element_class = mod_elem.DependentField
    elif etype in ('text_variable', 'text'):
        element_class = mod_elem.InputField
    elif etype in ('input_variable', 'input'):
        element_class = mod_elem.InputFieldInput
    elif etype in ('date_variable', 'date'):
        element_class = mod_elem.InputFieldDate
    elif etype in ('dropdown_variable', 'combo_variable', 'combo', 'dropdown'):
        element_class = mod_elem.InputFieldCombo
    elif etype in ('multiline_variable', 'multiline', 'multi'):
        element_class = mod_elem.InputFieldMultiline
    elif etype in ('checkbox', 'check'):
        element_class = mod_elem.InputFieldCheckbox
    else:
        raise AttributeError('unknown element type {ETYPE} provided to element {ELEM}'
                             .format(ETYPE=etype, ELEM=name))

    # Initialize the record element
    try:
        element = element_class(name, entry, parent=parent)
    except Exception as e:
        raise AttributeError('failed to initialize element {ELEM} - {ERR}'
                             .format(ELEM=name, ERR=e))

    return element