#! /usr/bin/env python3

"""Collect the corresponding information of nic"""

import os
import shutil
import argparse
from subprocess import Popen, PIPE

DEV_LIST = ["0x4000", "0x6000", "0x3800"]
VENDOR_LIST = ["0x19ee", "0x1da8"]
CONFIG_DEBUG_SCMD = False
SCRIPT_LOG = ""
BSP_PATHS = ["/opt/corigine/bin", "/opt/netronome/bin"]
REQ_SHELL_TOOLS = ["tar"]

# Map of island number to human readable component
# isl_nr: (name, nr_me's)
ISL_MAP = {
    1: ("ARM", 4),
    4: ("PCIe0", 4),
    5: ("PCIe1", 4),
    6: ("PCIe2", 4),
    7: ("PCIe3", 4),
    8: ("NBI0", 0),
    9: ("NBI1", 0),
    12: ("Crypto0", 8),
    13: ("Crypto1", 8),
    24: ("ExternalMUEng0", 0),
    25: ("ExternalMUEng1", 0),
    26: ("ExternalMUEng2", 0),
    28: ("InternalMUEng0", 0),
    29: ("InternalMUEng1", 0),
    32: ("MECl0", 12),
    33: ("MECl1", 12),
    34: ("MECl2", 12),
    35: ("MECl3", 12),
    36: ("MECl4", 12),
    37: ("MECl5", 12),
    38: ("MECl6", 12),
    48: ("Interlaken-LA0", 4),
    49: ("Interlaken-LA1", 4),
    }

rtsyms_to_dump = [
    "BLM_0_ISLAND_ID",
    "MAP_CMSG_Q_BASE",
    "SLICC_HASH_PAD_DATA",
    "_BLM_NBI8_BLQ0_EMU_QD_BASE",
    "_BLM_NBI8_BLQ0_EMU_Q_BASE",
    "_BLM_NBI8_BLQ1_EMU_QD_BASE",
    "_BLM_NBI8_BLQ1_EMU_Q_BASE",
    "_BLM_NBI8_BLQ2_EMU_QD_BASE",
    "_BLM_NBI8_BLQ2_EMU_Q_BASE",
    "_BLM_NBI8_BLQ3_EMU_QD_BASE",
    "_BLM_NBI8_BLQ3_EMU_Q_BASE",
    "_NIC_CFG_MCAST_INSTR_TBL",
    "__pv_pkt_sequencer",
    "_abi_cfg_lut_error_cnt",
    "_abi_dcb_cfg",
    "_abi_nfd_out_red_offload_0",
    "_abi_pcie_error_cnt",
    "_cfg_error_rss_cntr",
    "_fl_buf_sz_cache",
    "_gro_bm_0_0",
    "_gro_bm_0_1",
    "_gro_bm_0_2",
    "_gro_bm_0_3",
    "_gro_bm_1_0",
    "_gro_bm_1_1",
    "_gro_bm_1_2",
    "_gro_bm_1_3",
    "_gro_bm_2_0",
    "_gro_bm_2_1",
    "_gro_bm_2_2",
    "_gro_bm_2_3",
    "_gro_bm_3_0",
    "_gro_bm_3_1",
    "_gro_bm_3_2",
    "_gro_bm_3_3",
    "_gro_cli_cntrs",
    "_gro_global_config",
    "_gro_q_0_0",
    "_gro_q_0_1",
    "_gro_q_0_2",
    "_gro_q_0_3",
    "_gro_q_1_0",
    "_gro_q_1_1",
    "_gro_q_1_2",
    "_gro_q_1_3",
    "_gro_q_2_0",
    "_gro_q_2_1",
    "_gro_q_2_2",
    "_gro_q_2_3",
    "_gro_q_3_0",
    "_gro_q_3_1",
    "_gro_q_3_2",
    "_gro_q_3_3",
    "_mac_lkup_tbl",
    "_mac_stats",
    "_multi_pf_natqs_mask",
    "_multi_pf_of_vid",
    "_multi_pf_vf_alloc",
    "_nfd_cfg_multi_pf_state",
    "_nfd_stats_in_recv",
    "_nfd_stats_out_drop",
    "_nfd_stats_out_sent",
    "_nic_cfg_synch",
    "_nic_stats_queue",
    "_nic_stats_vnic",
    "_nic_vlan_to_vnics_map_tbl",
    "_no_mcast_filter_cache",
    "_pf0_net_app_cap",
    "_pf0_net_app_id",
    "_pf0_net_bar0",
    "_pf0_net_ctrl_bar",
    "_pf0_net_vf_cfg2",
    "_pf1_net_app_cap",
    "_pf1_net_app_id",
    "_pf1_net_bar0",
    "_pf1_net_vf_cfg2",
    "blm_pci4_blq_stash",
    "cmsg_dbg_enq__cntr__",
    "cmsg_dbg_rxq__cntr__",
    "cmsg_enq__cntr__",
    "cmsg_err__cntr__",
    "cmsg_err_no_credits__cntr__",
    "cmsg_rx__cntr__",
    "cmsg_rx_bad_type__cntr__",
    "cmsg_tx__cntr__",
    "debugdebug_journal",
    "fl_cache_mem0",
    "gro_release_ring_mem_0",
    "gro_release_ring_mem_1",
    "gro_release_ring_mem_2",
    "gro_release_ring_mem_3",
    "i12.NIC_CFG_INSTR_TBL_NBI",
    "i12.NIC_CFG_INSTR_TBL_NFD",
    "i12.NIC_RSS_TBL",
    "i12._feature_counters",
    "i32.NIC_CFG_INSTR_TBL_NBI",
    "i32.NIC_CFG_INSTR_TBL_NFD",
    "i32.NIC_RSS_TBL",
    "i32._feature_counters",
    "i33.NIC_CFG_INSTR_TBL_NBI",
    "i33.NIC_CFG_INSTR_TBL_NFD",
    "i33.NIC_RSS_TBL",
    "i33._feature_counters",
    "i34.BLM_BLQ0_CACHE_BASE",
    "i34.BLM_BLQ1_CACHE_BASE",
    "i34.BLM_BLQ2_CACHE_BASE",
    "i34.BLM_BLQ3_CACHE_BASE",
    "i34.BLM_EGRESS_NULL_BUF_RECYCLE_BASE_0",
    "i34.BLM_INFO_SECTION_BASE",
    "i34.BLM_INGRESS_NULL_BUF_RECYCLE_BASE_0",
    "i34.CTM_BLQ0_STATS_BASE",
    "i34.CTM_BLQ1_STATS_BASE",
    "i34.CTM_BLQ2_STATS_BASE",
    "i34.CTM_BLQ3_STATS_BASE",
    "i34.CTM_NBI_BLQ0_STATS_BASE",
    "i34.CTM_NBI_BLQ1_STATS_BASE",
    "i34.CTM_NBI_BLQ2_STATS_BASE",
    "i34.CTM_NBI_BLQ3_STATS_BASE",
    "i34.NIC_CFG_INSTR_TBL_NBI",
    "i34.NIC_CFG_INSTR_TBL_NFD",
    "i34.NIC_RSS_TBL",
    "i34._feature_counters",
    "i34._nvnic_macs",
    "i34.me10._multicast_addr_cntrs",
    "i34.me10._queue_stat_error_flag",
    "i34.me10._vf_cfg_rate",
    "i35.NIC_CFG_INSTR_TBL_NBI",
    "i35.NIC_CFG_INSTR_TBL_NFD",
    "i35.NIC_RSS_TBL",
    "i35._feature_counters",
    "i35._pkt_buf_ctm_credits",
    "i36.NIC_CFG_INSTR_TBL_NBI",
    "i36.NIC_CFG_INSTR_TBL_NFD",
    "i36.NIC_RSS_TBL",
    "i36._feature_counters",
    "i36._pkt_buf_ctm_credits",
    "i36.me10._msix_cls_automask",
    "i36.me10._msix_cls_rx_enabled",
    "i36.me10._msix_cls_rx_entries",
    "i36.me10._msix_cls_rx_new_enabled",
    "i36.me10._msix_cls_tx_enabled",
    "i36.me10._msix_cls_tx_entries",
    "i36.me10._msix_cls_tx_new_enabled",
    "i36.me10._msix_rx_irqc_cfg",
    "i36.me10._msix_rx_irqc_state",
    "i36.me10._msix_tx_irqc_cfg",
    "i36.me10._msix_tx_irqc_state",
    "i4._lso_hdr_data",
    "i4._nfd_in_issued_ring0",
    "i4._nfd_in_issued_ring1",
    "i4.me7._msix_cls_automask",
    "i4.me7._msix_cls_rx_enabled",
    "i4.me7._msix_cls_rx_entries",
    "i4.me7._msix_cls_rx_new_enabled",
    "i4.me7._msix_cls_tx_enabled",
    "i4.me7._msix_cls_tx_entries",
    "i4.me7._msix_cls_tx_new_enabled",
    "i4.me7._msix_cls_txr_wb_enabled",
    "i4.me7._msix_rx_irqc_cfg",
    "i4.me7._msix_rx_irqc_state",
    "i4.me7._msix_tx_irqc_cfg",
    "i4.me7._msix_tx_irqc_state",
    "i4.me7._msix_txr_wb_addr",
    "i4.nfd_active_bmsk_atomic",
    "i4.nfd_in_active_bmsk",
    "i4.nfd_in_batch_mem",
    "i4.nfd_in_gpd_wq_mem0",
    "i4.nfd_out_active_bmsk",
    "i4.nfd_out_active_bmsk_atomic",
    "i4.nfd_out_sb_wq_credits0",
    "i4.nfd_out_sb_wq_mem0",
    "i4.nfd_queue_event_mem0",
    "i48.BLM_BLQ0_CACHE_BASE",
    "i48.BLM_BLQ1_CACHE_BASE",
    "i48.BLM_BLQ2_CACHE_BASE",
    "i48.BLM_BLQ3_CACHE_BASE",
    "i48.BLM_EGRESS_NULL_BUF_RECYCLE_BASE_0",
    "i48.BLM_INFO_SECTION_BASE",
    "i48.BLM_INGRESS_NULL_BUF_RECYCLE_BASE_0",
    "i48.CTM_BLQ0_STATS_BASE",
    "i48.CTM_BLQ1_STATS_BASE",
    "i48.CTM_BLQ2_STATS_BASE",
    "i48.CTM_BLQ3_STATS_BASE",
    "i48.CTM_NBI_BLQ0_STATS_BASE",
    "i48.CTM_NBI_BLQ1_STATS_BASE",
    "i48.CTM_NBI_BLQ2_STATS_BASE",
    "i48.CTM_NBI_BLQ3_STATS_BASE",
    "i48.nfd_out_sb_wq_credits0",
    "i48.nfd_out_sb_wq_mem0",
    "mem_meter",
    "mem_meter_cfg",
    "nbi0_dma_bpe_credits",
    "nfd_cfg_bar0_base0",
    "nfd_cfg_bar1_base0",
    "nfd_cfg_base0",
    "nfd_cfg_msg_jrnl",
    "nfd_cfg_pf0_num_ports",
    "nfd_cfg_pf1_num_ports",
    "nfd_cfg_ring_mem",
    "nfd_cfg_ring_mem00",
    "nfd_cfg_ring_mem01",
    "nfd_cfg_ring_mem02",
    "nfd_cfg_ring_mem03",
    "nfd_cfg_ring_mem04",
    "nfd_cfg_ring_mem05",
    "nfd_cfg_ring_mem06",
    "nfd_cfg_state_jrnl",
    "nfd_cfg_tlv_template0",
    "nfd_flr_atomic",
    "nfd_flr_seen",
    "nfd_in_cntrs0",
    "nfd_in_gather_debug_queue_state0",
    "nfd_in_gather_debug_state0",
    "nfd_in_halt_wq_mem",
    "nfd_in_pd_debug_state0",
    "nfd_in_tx_sent0",
    "nfd_init_done",
    "nfd_init_done_atomic",
    "nfd_out_atomics0",
    "nfd_out_cacher_dbg0",
    "nfd_out_cacher_qdbg0",
    "nfd_out_cntrs0",
    "nfd_out_ring_mem0",
    "nfd_out_sb_debug_state0",
    "nfd_out_sb_dma_done0",
    "nfd_out_sb_release0",
    "nfd_vf_cfg_max_vfs",
    "num_ov_alloc__cntr__",
    "num_ov_free__cntr__",
    "pkt_counters_base0",
    "pkt_counters_base1",
    "pkt_counters_base2",
    "pkt_counters_base3",
    "pkt_counters_base4",
    "pkt_counters_base5",
    "pkt_counters_base6",
    "sriov_journal_cmsg",
    "unroll_overflow_err__cntr__",
]

NbiXPB_to_dump = [
    "TrafficManager.SchedulerL1Deficit.SchedulerDeficit{0..1023}",
    "TrafficManager.SchedulerL1Weight.SchedulerWeight{0..1023}",
    "TrafficManager.SchedulerL2Deficit.SchedulerDeficit{0..1023}",
    "TrafficManager.SchedulerL2Weight.SchedulerWeight{0..1023}",

    "TrafficManager.TMQueueReg.QueueConfig{0..1023}",
    "TrafficManager.TMQueueReg.QueueDropCount{0..1023}",
    "TrafficManager.TMQueueReg.QueueStatus{0..1023}",

    "TrafficManager.TMSchedulerReg.SchedulerConfig{0..144}",
    "TrafficManager.TMSchedulerReg.SchedulerDeficit{0..127}",
    "TrafficManager.TMSchedulerReg.SchedulerWeight{0..127}",

    "TrafficManager.TMShaperReg.ShaperMaxOvershoot{0..144}",
    "TrafficManager.TMShaperReg.ShaperRate{0..144}",
    "TrafficManager.TMShaperReg.ShaperRateAdjust{0..144}",
    "TrafficManager.TMShaperReg.ShaperStatus{0..144}",
    "TrafficManager.TMShaperReg.ShaperThreshold{0..144}",

    "TrafficManager.TrafficManagerReg.BlqEvent",
    "TrafficManager.TrafficManagerReg.DropRate",
    "TrafficManager.TrafficManagerReg.EgressPullIdPortEnable",
    "TrafficManager.TrafficManagerReg.EgressRateLimit",
    "TrafficManager.TrafficManagerReg.MiniPktChannelCredit{0..127}",
    "TrafficManager.TrafficManagerReg.MiniPktCreditConfig",
    "TrafficManager.TrafficManagerReg.MiniPktFreePoolCredit0",
    "TrafficManager.TrafficManagerReg.MiniPktFreePoolCredit1",
    "TrafficManager.TrafficManagerReg.ReorderActivity",
    "TrafficManager.TrafficManagerReg.TrafficManagerConfig",

    "PktPreclassifier.Characterization.Config",
    "PktPreclassifier.Characterization.TableSet",
    "PktPreclassifier.Picoengine.PicoengineSetup",
    "PktPreclassifier.Picoengine.TableExtend",
    "PktPreclassifier.Picoengine.ActiveSet0Low",
    "PktPreclassifier.Picoengine.ActiveSet0High",
    "PktPreclassifier.Picoengine.ActiveSet1Low",
    "PktPreclassifier.Picoengine.ActiveSet1High",
    "PktPreclassifier.Picoengine.ClassifiedSmall",
    "PktPreclassifier.Picoengine.ClassifiedLarge",
    "PktPreclassifier.Picoengine.Tunnel",
    "PktPreclassifier.PolicerSequencer.Accumulator{0..7}",
    "PktPreclassifier.PolicerSequencer.Config",
    "PktPreclassifier.PolicerSequencer.Sequence{0..7}",
    ]

def parse_cmdline():
    """Use argparse to read command line variables."""
    parser = argparse.ArgumentParser(
        description='Collect various bits of debugging information')
    parser.add_argument(
        '-i',
        '--interface',
        dest='iface',
        nargs='?',
        action='store',
        default='',
        help="Optional - nfp netdev to collect information on")
    parser.add_argument(
        '-l',
        '--logdir',
        dest='logdir',
        action='store',
        default='/var/tmp',
        help="Optional - base directory to use for logs")

    args = parser.parse_args()

    # Verify arguments
    if args.iface:
        iface = args.iface
        if iface not in os.listdir("/sys/class/net"):
            print(f"{iface} not found on system")
            exit(1)
        with open(f"/sys/class/net/{iface}/device/vendor", "r") as vendor_f:
            vendor = vendor_f.read().strip()
            if vendor not in VENDOR_LIST:
                print(f"{iface} with vendor id: {vendor} is not "
                      "a valid nfp device")
                exit(1)

    return args

def get_nfps():
    """Return dict of NFP's on the system, key is the BDF."""
    # Walk /sys/bus/pci/devices - this removes the need to depend on
    # lspci being installed
    nfp_dict = {}
    devices = '/sys/bus/pci/devices'
    for root, dirs, files in os.walk(devices):
        for pci in dirs:
            with open(f"{root}/{pci}/vendor", "r") as vendor_f:
                vendor = vendor_f.read().strip()
                if vendor in VENDOR_LIST:
                    with open(f"{root}/{pci}/device", "r") as dev_f:
                        device = dev_f.read().strip()
                        if device in DEV_LIST:
                            nfp_dict[pci] = {}
    return nfp_dict

def check_bsp_tools_installed():
    """Check if bsp tools are installed on the system.
       Returns the path of the installation if found, else empty string
    """
    # Check if in path by checking "which"
    path = shutil.which("nfp-hwinfo")
    if path is not None:
        return os.path.dirname(path)

    # Otherwise check common path(s) for tools installation
    for path in BSP_PATHS:
        if os.path.exists(path) and "nfp-hwinfo" in os.listdir(path):
            return path

    return ""

def scmd(command, fail=True):
    """Run bash commands from python.

    Args:
        - command:    Command to execute
        - fail: Stops program if a command fails and prints debug info
    Return:
        A tuple with STDOUT, STDERR and the return code
    """
    global SCRIPT_LOG
    process = Popen(command, stdout=PIPE, stderr=PIPE, close_fds=False,
                    shell=True)

    stdoutdata, stderrdata = process.communicate()
    ret = process.returncode
    msg = f'CMD: {command}\n'
    msg += f'RET: {ret}\n'
    if ret != 0 and fail is True:
        msg += f'STDOUT:\n{stdoutdata.decode()}\n'
        msg += f'STDERR:\n{stderrdata.decode()}\n'
        if CONFIG_DEBUG_SCMD:
            print(msg)
        exit(1)
    if SCRIPT_LOG != '':
        with open(SCRIPT_LOG, 'a') as logf:
            logf.write(msg)

    return stdoutdata.decode(), stderrdata.decode(), ret

def bsp_comand(bsp_path, nfp_pci, bsp_cmd, options=""):
    """Helper to execute bsp commands"""
    cmd = f"{bsp_path}/{bsp_cmd} -Z {nfp_pci} {options}"
    out, err, ret = scmd(cmd, fail=False)
    return out, err, ret

def write_logfile(input_strings, filename, flags='w'):
    """Write strings log into corresponding file"""
    with open(filename, flags) as f:
        f.write(input_strings)

def common_data(outDir):
    """Output data that needs to be stored separately"""

    # Log firmware path where driver loads from. Could be symlinked files
    libFw, _, _ = scmd('ls -ogR --time-style="+" /lib/firmware/netronome/',
                       fail=False)
    libFw_path = f'{outDir}/lib_firmware.log'
    write_logfile(libFw, libFw_path)

    # Log firmware path where package installs places files
    optFw, _, _ = scmd('ls -ogR --time-style="+" /opt/firmware/netronome/',
                       fail=False)
    optFw_path = f'{outDir}/opt_firmware.log'
    write_logfile(optFw, optFw_path)

    nfp_pcis = get_nfps()
    nfp_pcis_path = f'{outDir}/nfp_pcis.log'
    write_logfile(str(nfp_pcis), nfp_pcis_path)

    for line in nfp_pcis:
        # Driver might not be bound to nfp, in which case none of the below
        # data can be gathered. Log this and move to next nfp pci device.
        if not os.path.exists(f'/sys/bus/pci/drivers/nfp/{line}/net'):
            no_ndev_log_path = f'{outDir}/{line}_no_netdevs'
            msg = f"No netdevs seem to exist for {line}. Most likely cause "
            msg += "is that the device is not bound to the nfp driver\n"
            write_logfile(msg, no_ndev_log_path)
            continue

        ndevs = "\n".join(os.listdir(f'/sys/bus/pci/drivers/nfp/{line}/net'))
        ndevs_path = f'{outDir}/ndevs.log'
        write_logfile(ndevs, ndevs_path, flags='a')
        ndev = os.listdir(f'/sys/bus/pci/drivers/nfp/{line}/net')[0]
        # set ethtool loglevel to 0
        scmd(f'ethtool -W {ndev} 0', fail=False)
        scmd(f'ethtool -w {ndev} data {outDir}/ethtool_{ndev}_debug_L0.dat',
             fail=False)
        # set ethtool loglevel to 1
        scmd(f'ethtool -W {ndev} 1', fail=False)
        scmd(f'ethtool -w {ndev} data {outDir}/ethtool_{ndev}_debug_L1.dat',
             fail=False)

    # devlink dev list
    dev_list, _, _ = scmd('devlink dev list', fail=False)
    dev_list_path = f'{outDir}/devlink_dev_list'
    write_logfile(dev_list, dev_list_path)

    with open(dev_list_path) as f:
        for port in f:
            port = port.replace('\n', '')
            pName = port.replace('/', '-')
            # devlink dev info <pci_address>
            dev_info, _, _ = scmd(f'devlink dev info {port}', fail=False)
            dev_info_path = f'{outDir}/devlink_dev_info_{pName}'
            write_logfile(dev_info, dev_info_path)

    # sensors
    sensors_info, _, _ = scmd('sensors -n', fail=False)
    sensors_path = f'{outDir}/sensors'
    write_logfile(sensors_info, sensors_path)

    # lspci
    lspci_info, _, _ = scmd('lspci -vvv', fail=False)
    lspci_path = f'{outDir}/lspci'
    write_logfile(lspci_info, lspci_path)

    # lspci - tree view
    lspci_tv_info, _, _ = scmd('lspci -tv', fail=False)
    lspci_tv_path = f'{outDir}/lspci_tv'
    write_logfile(lspci_tv_info, lspci_tv_path)

    # firewalld
    firewalld_info, _, _ = scmd('systemctl status firewalld', fail=False)
    firewalld_path = f'{outDir}/systemctl_status_firewalld'
    write_logfile(firewalld_info, firewalld_path)

    # irqbalance
    irqbalance_info, _, _ = scmd('systemctl status irqbalance', fail=False)
    irqbalance_path = f'{outDir}/systemctl_status_irqbalance'
    write_logfile(irqbalance_info, irqbalance_path)

    # lscpu
    lscpu_info, _, _ = scmd('lscpu', fail=False)
    lscpu_path = f'{outDir}/lscpu'
    write_logfile(lscpu_info, lscpu_path)

    # lshw
    lshw_info, _, _ = scmd('lshw -class network -businfo', fail=False)
    lshw_path = f'{outDir}/lshw'
    write_logfile(lshw_info, lshw_path)

    # /etc/*release
    etc_release_info, _, _ = scmd('ls -l /etc/*release && cat /etc/*release',
                                  fail=False)
    etc_release_path = f'{outDir}/etc_release'
    write_logfile(etc_release_info, etc_release_path)

    # numactl --hardware
    numactl_info, _, _ = scmd('numactl --hardware', fail=False)
    numactl_path = f'{outDir}/numactl_hardware'
    write_logfile(numactl_info, numactl_path)

    # The next two commands tries to capture the installed packages on
    # the system. Adding system detection that is stable accross all
    # OS's is complicated and error prone, so instead just log both
    # std_out and std_err for both 'rpm -qa' and 'dpkg -l'

    # rpm -qa
    rpm_info, err, _ = scmd('rpm -qa', fail=False)
    rpm_info += f"\n{err}"
    rpm_path = f'{outDir}/rpm_package_list'
    write_logfile(rpm_info, rpm_path)

    # dpkg -l
    deb_info, err, _ = scmd('dpkg -l', fail=False)
    deb_info += f"\n{err}"
    deb_path = f'{outDir}/deb_package_list'
    write_logfile(deb_info, deb_path)

    # dmidecode
    for dmi in ["bios", "system", "baseboard", "chassis", "processor",
                "memory", "cache", "connector", "slot"]:

        dmi_info, _, _ = scmd(f'dmidecode -t {dmi}', fail=False)
        dmi_path = f'{outDir}/dmidecode_{dmi}'
        write_logfile(dmi_info, dmi_path)

    # /proc/meminfo
    meminfo_path = f'{outDir}/proc_meminfo'
    try:
        shutil.copy2('/proc/meminfo', meminfo_path)
    except Exception:
        write_logfile('Error reading /proc/meminfo', meminfo_path)

    # /proc/cpuinfo
    cpuinfo_path = f'{outDir}/proc_cpuinfo'
    try:
        shutil.copy2('/proc/cpuinfo', cpuinfo_path)
    except Exception:
        write_logfile('Error reading /proc/cpuinfo', cpuinfo_path)

    # /var/boot.log
    bootlog_path = f'{outDir}/boot.log'
    try:
        shutil.copy2('/var/log/boot.log', bootlog_path)
    except Exception:
        write_logfile('Error reading /var/log/boot.log', bootlog_path)

    # history
    home_dir = os.path.expanduser('~')
    history_infile = f'{home_dir}/.bash_history'
    history_outfile = f'{outDir}/history'
    try:
        shutil.copy2(history_infile, history_outfile)
    except Exception:
        write_logfile(f'{history_infile}/ not found',
                      history_outfile)

    # /etc/udev/rules.d
    udev_indir = '/etc/udev/rules.d'
    udev_outdir = f'{outDir}/etc_udev_rules.d'
    try:
        shutil.copytree(udev_indir, udev_outdir)
    except Exception:
        os.makedirs(udev_outdir, exist_ok=True)
        write_logfile(f'Could not backup {udev_indir}\n',
                      f'{udev_outdir}/udev_rules.d_err')

    # /usr/lib/udev/rules.d
    udev_indir = '/usr/lib/udev/rules.d'
    udev_outdir = f'{outDir}/usr_lib_udev_rules.d'
    try:
        shutil.copytree(udev_indir, udev_outdir)
    except Exception:
        os.makedirs(udev_outdir, exist_ok=True)
        write_logfile(f'Could not backup {udev_indir}\n',
                      f'{udev_outdir}/usr_udev_rules.d_err')

    # /etc/sysconfig/network-scripts
    nscripts_indir = '/etc/sysconfig/network-scripts'
    nscripts_outdir = f'{outDir}/network-scripts'
    try:
        shutil.copytree(nscripts_indir, nscripts_outdir)
    except Exception:
        os.makedirs(nscripts_outdir, exist_ok=True)
        write_logfile(f'Could not backup {nscripts_indir}\n',
                      f'{nscripts_outdir}/nscripts_err')

def sos_data(outDir):
    """Output data from sos."""
    sos_cmd = get_sosreport_cmd()
    if sos_cmd == "":
        return False
    loaded_plugins = "-o networking -o logs -o kernel"
    loaded_plugins_path = f'{outDir}/loaded_plugins'
    write_logfile(f"{loaded_plugins}\n", loaded_plugins_path)
    # sos report -o networking -o logs -o kernel --batch --tmp-dir --build
    print("Generate DATA with sos")
    scmd(f'{sos_cmd} {loaded_plugins} --batch --tmp-dir {outDir} --build',
         fail=False)
    return True

def self_data(outDir):
    """Output data similar to sos but add self"""
    ethDir = f'{outDir}/ethtool'
    os.makedirs(ethDir, exist_ok=True)
    # ls /sys/class/net
    net_list, _, _ = scmd('ls /sys/class/net', fail=False)
    net_list_path = f'{ethDir}/net_list'
    write_logfile(net_list, net_list_path)

    with open(net_list_path) as f:
        for item in f:
            item = item.replace('\n', '')
            # ethtool <port>
            ethtool, _, _ = scmd(f'ethtool {item}', fail=False)
            ethtool_path = f'{ethDir}/ethtool_{item}'
            write_logfile(ethtool, ethtool_path)
            # ethtool -a <port>
            ethtool_a, _, _ = scmd(f'ethtool -a {item}', fail=False)
            ethtool_a_path = f'{ethDir}/ethtool_-a_{item}'
            write_logfile(ethtool_a, ethtool_a_path)
            # ethtool -c <port>
            ethtool_c, _, _ = scmd(f'ethtool -c {item}', fail=False)
            ethtool_c_path = f'{ethDir}/ethtool_-c_{item}'
            write_logfile(ethtool_c, ethtool_c_path)
            # ethtool -d <port>
            ethtool_d, _, _ = scmd(f'ethtool -d {item}', fail=False)
            ethtool_d_path = f'{ethDir}/ethtool_-d_{item}'
            write_logfile(ethtool_d, ethtool_d_path)
            # ethtool -k <port>
            ethtool_k, _, _ = scmd(f'ethtool -k {item}', fail=False)
            ethtool_k_path = f'{ethDir}/ethtool_-k_{item}'
            write_logfile(ethtool_k, ethtool_k_path)
            # ethtool -i <port>
            ethtool_i, _, _ = scmd(f'ethtool -i {item}', fail=False)
            ethtool_i_path = f'{ethDir}/ethtool_-i_{item}'
            write_logfile(ethtool_i, ethtool_i_path)
            # ethtool -P <port>
            ethtool_P, _, _ = scmd(f'ethtool -P {item}', fail=False)
            ethtool_P_path = f'{ethDir}/ethtool_-P_{item}'
            write_logfile(ethtool_P, ethtool_P_path)
            # ethtool --phy-statistics <port>
            ethtool_phy_s, _, _ = scmd(f'ethtool --phy-statistics {item}',
                                       fail=False)
            ethtool_phy_s_path = f'{ethDir}/ethtool_-phy-statistics_{item}'
            write_logfile(ethtool_phy_s, ethtool_phy_s_path)
            # ethtool -S <port>
            ethtool_S, _, _ = scmd(f'ethtool -S {item}', fail=False)
            ethtool_S_path = f'{ethDir}/ethtool_-S_{item}'
            write_logfile(ethtool_S, ethtool_S_path)
            # ethtool -g <port>
            ethtool_g, _, _ = scmd(f'ethtool -g {item}', fail=False)
            ethtool_g_path = f'{ethDir}/ethtool_-g_{item}'
            write_logfile(ethtool_g, ethtool_g_path)
            # ethtool -T <port>
            ethtool_T, _, _ = scmd(f'ethtool -T {item}', fail=False)
            ethtool_T_path = f'{ethDir}/ethtool_-T_{item}'
            write_logfile(ethtool_T, ethtool_T_path)
            # ethtool -m <port>
            ethtool_m, _, _ = scmd(f'ethtool -m {item}', fail=False)
            ethtool_m_path = f'{ethDir}/ethtool_-m_{item}'
            write_logfile(ethtool_m, ethtool_m_path)
            # ethtool --show-eee <port>
            ethtool_show_eee, _, _ = scmd(f'ethtool --show-eee {item}',
                                          fail=False)
            ethtool_show_eee_path = f'{ethDir}/ethtool_-show-eee_{item}'
            write_logfile(ethtool_show_eee, ethtool_show_eee_path)
            # ethtool --show-fec <port>
            ethtool_show_fec, _, _ = scmd(f'ethtool --show-fec {item}',
                                          fail=False)
            ethtool_show_fec_path = f'{ethDir}/ethtool_-show-fec_{item}'
            write_logfile(ethtool_show_fec, ethtool_show_fec_path)
            # ethtool --show-priv-flags <port>
            ethtool_show_priv, _, _ = scmd(f'ethtool --show-priv-flags {item}',
                                           fail=False)
            ethtool_priv_path = f'{ethDir}/ethtool_-show-priv-flags_{item}'
            write_logfile(ethtool_show_priv, ethtool_priv_path)
            # tc -s filter show dev <port>
            tc_filter, _, _ = scmd(f'tc -s filter show dev {item}', fail=False)
            tc_filter_path = f'{ethDir}/tc_-s_filter_{item}'
            write_logfile(tc_filter, tc_filter_path)
            # tc -s filter show dev <port> ingress
            tc_ingress, _, _ = scmd(f'tc -s filter show dev {item} ingress',
                                    fail=False)
            tc_ingress_path = f'{ethDir}/tc_-s_filter_{item}_ingress'
            write_logfile(tc_ingress, tc_ingress_path)

    # nstat -zas
    nstat_zas, _, _ = scmd('nstat -zas', fail=False)
    nstat_zas_path = f'{ethDir}/nstat_-zas'
    write_logfile(nstat_zas, nstat_zas_path)
    # netstat -W -neopa
    netstat_W_neopa, _, _ = scmd('netstat -W -neopa', fail=False)
    netstat_W_neopa_path = f'{ethDir}/netstat_-W_-neopa'
    write_logfile(netstat_W_neopa, netstat_W_neopa_path)
    # netstat -W -agn
    netstat_W_agn, _, _ = scmd('netstat -W -agn', fail=False)
    netstat_W_agn_path = f'{ethDir}/netstat_-W_-agn'
    write_logfile(netstat_W_agn, netstat_W_agn_path)
    # netstat -s
    netstat_s, _, _ = scmd('netstat -s', fail=False)
    netstat_s_path = f'{ethDir}/netstat_s'
    write_logfile(netstat_s, netstat_s_path)
    # ip -s neigh show
    ip_s_neigh_show, _, _ = scmd('ip -s neigh show', fail=False)
    ip_s_neigh_show_path = f'{ethDir}/ip_-s_neigh_show'
    write_logfile(ip_s_neigh_show, ip_s_neigh_show_path)
    # ip -s -d link
    ip_sd_link, _, _ = scmd('ip -s -d link', fail=False)
    ip_sd_link_path = f'{ethDir}/ip_-s_-d_link'
    write_logfile(ip_sd_link, ip_sd_link_path)
    # ip route show table all
    ip_r_show_tb, _, _ = scmd('ip route show table all', fail=False)
    ip_r_show_tb_path = f'{ethDir}/ip_route_show_table_all'
    write_logfile(ip_r_show_tb, ip_r_show_tb_path)
    # ip -o addr
    ip_o_addr, _, _ = scmd('ip -o addr', fail=False)
    ip_o_addr_path = f'{ethDir}/ip_-o_addr'
    write_logfile(ip_o_addr, ip_o_addr_path)
    # ip netns
    ip_netns, _, _ = scmd('ip netns', fail=False)
    ip_netns_path = f'{ethDir}/ip_netns'
    write_logfile(ip_netns, ip_netns_path)
    # ip neigh show nud noarp
    ip_nshow_nud, _, _ = scmd('ip neigh show nud noarp', fail=False)
    ip_nshow_nud_path = f'{ethDir}/ip_neigh_show_nud_noarp'
    write_logfile(ip_nshow_nud, ip_nshow_nud_path)
    # ip mroute show
    ip_mroute, _, _ = scmd('ip mroute show', fail=False)
    ip_mroute_path = f'{ethDir}/ip_mroute_show'
    write_logfile(ip_mroute, ip_mroute_path)
    # ip maddr show
    ip_maddr, _, _ = scmd('ip maddr show', fail=False)
    ip_maddr_path = f'{ethDir}/ip_maddr_show'
    write_logfile(ip_maddr, ip_maddr_path)
    # ip -d route show cache
    ip_droute, _, _ = scmd('ip -d route show cache', fail=False)
    ip_droute_path = f'{ethDir}/ip_-d_route_show_cache'
    write_logfile(ip_droute, ip_droute_path)
    # ip -d address
    ip_daddr, _, _ = scmd('ip -d address', fail=False)
    ip_daddr_path = f'{ethDir}/ip_-d_address'
    write_logfile(ip_daddr, ip_daddr_path)
    # ip -d -6 route show cache
    ip_d6route, _, _ = scmd('ip -d -6 route show cache', fail=False)
    ip_d6route_path = f'{ethDir}/ip_-d_-6_route_show_cache'
    write_logfile(ip_d6route, ip_d6route_path)
    # ip -6 rule
    ip_6_rule, _, _ = scmd('ip -6 rule', fail=False)
    ip_6_rule_path = f'{ethDir}/ip_-d_-6_route_show_cache'
    write_logfile(ip_6_rule, ip_6_rule_path)
    # ip -6 route show table all
    ip_6_route_tb, _, _ = scmd('ip -6 route show table all', fail=False)
    ip_6_route_tb_path = f'{ethDir}/ip_-6_route_show_table_all'
    write_logfile(ip_6_route_tb, ip_6_route_tb_path)
    # ip -4 rule
    ip_4_rule, _, _ = scmd('ip -4 rule', fail=False)
    ip_4_rule_path = f'{ethDir}/ip_-4_rule'
    write_logfile(ip_4_rule, ip_4_rule_path)
    # ifenslave -a
    ifenslave_a, _, _ = scmd('ifenslave -a', fail=False)
    ifenslave_a_path = f'{ethDir}/ifenslave_-a'
    write_logfile(ifenslave_a, ifenslave_a_path)

    knlDir = f'{outDir}/kernel'
    os.makedirs(knlDir, exist_ok=True)
    # cat /etc/hostname
    etc_hostname, _, _ = scmd('cat /etc/hostname', fail=False)
    etc_hostname_path = f'{knlDir}/etc_hostname.log'
    write_logfile(etc_hostname, etc_hostname_path)
    # journalctl -k
    journalctl_k, _, _ = scmd('journalctl -k', fail=False)
    journalctl_k_path = f'{knlDir}/journalctl_-k.log'
    write_logfile(journalctl_k, journalctl_k_path)
    # uname -a
    uname_a, _, _ = scmd('uname -a', fail=False)
    uname_a_path = f'{knlDir}/uname_-a.log'
    write_logfile(uname_a, uname_a_path)
    # cat /etc/*release
    etc_release, _, _ = scmd('cat /etc/*release', fail=False)
    etc_release_path = f'{knlDir}/etc_release.log'
    write_logfile(etc_release, etc_release_path)
    # modinfo nfp
    modinfo_nfp, _, _ = scmd('modinfo nfp', fail=False)
    modinfo_nfp_path = f'{knlDir}/modinfo_nfp.log'
    write_logfile(modinfo_nfp, modinfo_nfp_path)
    # lsmod
    lsmod, _, _ = scmd('lsmod', fail=False)
    lsmod_path = f'{knlDir}/lsmod.log'
    write_logfile(lsmod, lsmod_path)
    # cat /proc/cmdline
    cmdline, _, _ = scmd('cat /proc/cmdline', fail=False)
    cmdline_path = f'{knlDir}/cmdline.log'
    write_logfile(cmdline, cmdline_path)
    # ls /sys/bus/pci/drivers/nfp/module/parameters/
    oot_para_path = '/sys/bus/pci/drivers/nfp/module/parameters/'
    oot_parameters, _, _ = scmd(f'ls {oot_para_path}', fail=False)
    oot_parameters_path = f'{knlDir}/oot_parameters.log'
    write_logfile(oot_parameters, oot_parameters_path)
    # cat /boot/config-$(uname -r)
    kernel_config, _, _ = scmd('cat /boot/config-$(uname -r)', fail=False)
    kernel_config_path = f'{knlDir}/kernel_config.log'
    write_logfile(kernel_config, kernel_config_path)

    if 'CentOS' in etc_release:
        # lsinitrd
        lsinitrd, _, _ = scmd('lsinitrd', fail=False)
        lsinitrd_path = f'{knlDir}/lsinitrd.log'
        write_logfile(lsinitrd, lsinitrd_path)
    elif 'Ubuntu' in etc_release:
        # sinitramfs /boot/initrd.img
        lsinitramfs, _, _ = scmd('lsinitramfs /boot/initrd.img', fail=False)
        lsinitramfs_path = f'{knlDir}/lsinitramfs.log'
        write_logfile(lsinitramfs, lsinitramfs_path)

def check_cpp_access(outDir):
    """Checks if the driver was loaded with cpp access enabled"""
    cpp_param = 0
    bspDir = f'{outDir}/bsp_logs'
    os.makedirs(bspDir, exist_ok=True)

    nfp_module_path = '/sys/bus/pci/drivers/nfp/module'
    cpp_param_path = f'{nfp_module_path}/parameters/nfp_dev_cpp'

    # Check if in-tree driver is loaded
    if not os.path.exists(cpp_param_path):
        fail_msg_path = f'{bspDir}/fail_intree_module'
        message = f'{nfp_module_path} does not exist\n'
    else:
        with open(cpp_param_path) as cpp_file:
            cpp_param = cpp_file.read()
        cpp_param = int(cpp_param)
        if cpp_param == 0:
            fail_msg_path = f'{bspDir}/fail_cpp_not_enabled'
            message = "driver not loaded with cpp_access enabled\n"

    # Check if driver is loaded at all, and override error
    # message if not loaded
    if not os.path.exists(nfp_module_path):
        fail_msg_path = f'{bspDir}/fail_module_not_loaded'
        message = f'{nfp_module_path} missing, driver not loaded or built-in\n'

    # If cpp_param did not get set write to log and exit
    if cpp_param == 0:
        write_logfile(message, fail_msg_path)
        return False
    return True

def filter_multi_pf(pci_list):
    """Returns a filtered list, containing only single pci-address
    for every card.

    Entries in list is in the normal DBDF <0000:aa:00.0>.
    If there are multi-PF devices then there will be two (or more)
    entries for a card, e.g:
        0000:aa:00.0
        0000:aa:00.1

    This function will return a list containing only one entry per
    card.
    """

    # This will be the new returned list
    filt_pci = []
    # This list is temporary to keep track of previously seen devices
    BDF_list = []

    for pci in pci_list:
        BDF = pci.split('.')[0]
        if BDF not in BDF_list:
            BDF_list.append(BDF)
            filt_pci.append(pci)
    return filt_pci

def netdev_to_pci(netdev):
    """Takes in a netdev name, and returns an pci-address if possible,
    otherwise None
    """
    try:
        pci_path = os.readlink(f'/sys/class/net/{netdev}/device')
        pci_path = os.path.basename(pci_path)
        return pci_path
    except Exception:
        return None

def get_enabled_islands(bsp_path, pci):
    """Parses 'chip.island' and return list of enabled islands"""

    isl_list = []
    out, _, _ = bsp_comand(bsp_path, pci, 'nfp-hwinfo', options='chip.island')
    isl_hex_str = out.split('=')[1]
    # isl_hex_str in format '0x1001f13000112'
    isl_int = int(isl_hex_str, 16)

    # Bit positions correspond with enabled islands, convert to binary
    # representation in order to parse
    isl_bin_str = "{:b}".format(isl_int)
    for nr, me in enumerate(isl_bin_str[::-1]):
        if me == '1':
            isl_list.append(nr)
    return isl_list

def parse_rtsym_table(sym_str):
    """Parses the output of the 'nfp-rtsym -L' command
        Name                    Resource   Address      Size
        BLM_EMU_RING_INIT_CHK_0 <generic>  0x0000000000 0x0000000001
        nfd_in_ring_nums0       <generic>  0x0000000000 0x0000000001

    Returns a dictionary of dictionaries:
        { "BLM_EMU..." :
            { "Resource": <generic>,
              "Address": 0x000...,
              "Size": int(0x000...),
            },
          "nfd_in...":
          {
          }
        }
    """
    rtsym_dict = {}
    # Iterate through lines, but skip the heading
    for line in sym_str.strip().split("\n")[1:]:
        name, res, addr, size = line.strip().split()
        rtsym_dict[name] = {
            "Resource": res,
            "Address": addr,
            "Size": int(size, 16)
            }
    return rtsym_dict

def filter_xpb_nbi(xpb_in):
    """Dumping ALL the nbi xpb/m information takes very long and does
    sometime leave the card in a bad state. Limit to only specific
    information and return a list of commands.
    """
    nbi_cmd_list = []
    for nbiXPB in NbiXPB_to_dump:
        cmd = f'{xpb_in}NbiTopXpbMap.{nbiXPB}'
        nbi_cmd_list.append(cmd)

    return nbi_cmd_list

def write_rtsym(bsp_path, pci, rtsym_dict, key, logpath):
    """Simple wrapper to log rtsym command"""
    head = f'rtsym={key},size={rtsym_dict[key]["Size"]}\n'
    write_logfile(head, logpath, "a")
    dat, _, _ = bsp_comand(bsp_path, pci, 'nfp-rtsym', options=f'-v {key}')
    write_logfile(dat, logpath, "a")

def collect_bsp_data(bsp_path, outDir, args):
    """Collect low level debug information using BSP tools"""

    bspDir = f"{outDir}/bsp_logs"
    os.makedirs(bspDir, exist_ok=True)
    all_nfp_pcis = get_nfps()
    nfp_pcis = filter_multi_pf(all_nfp_pcis)

    # Collect generic BSP info for all nfp's
    for pci in nfp_pcis:
        # Each entry a tuple of the command to run [0], and the option
        # flags to use [1]
        cmds = (
            ('nfp-arm', '-D'),
            ('nfp-fis', '-b1 list'),
            ('nfp-hwinfo', ''),
            ('nfp-media', ''),
            ('nfp-nsp', '-E'),
            ('nfp-phymod', ''),
            ('nfp-res', '-L'),
            ('nfp-temp', ''),
            ('nic-power', ''),
        )
        for cmd, opt in cmds:
            out, _, _ = bsp_comand(bsp_path, pci, cmd, options=opt)
            fpath = f"{bspDir}/{pci}_{cmd}"
            # If option is specified add this to the output filename
            if opt:
                fpath = f"{fpath}_{opt}"
            out_str = f"{out.strip()}\n"
            write_logfile(out_str, fpath, "w")

        # Log the enabled NFP islands
        isls = get_enabled_islands(bsp_path, pci)
        fpath = f"{bspDir}/{pci}_islands"
        out_str = (",".join(ISL_MAP[x][0] for x in isls))
        out_str += "\n"
        write_logfile(out_str, fpath, "w")

        # Check if gearbox is in use, and log additional info
        hwinfo, _, _ = bsp_comand(bsp_path, pci, 'nfp-hwinfo', options='')
        hwinfo_d = {}
        for line in hwinfo.strip().split('\n'):
            key, val = line.strip().split('=')
            hwinfo_d[key] = val

        if 'phy0.mvl5113' in hwinfo_d.keys():
            # Get nr of ports:
            port_count = 0
            for k in hwinfo_d.keys():
                if '.nbi' in k:
                    port_count += 1

            # Log mdio commands
            fpath = f"{bspDir}/{pci}_mdio"
            cmd_list = [
                '7.0000',  # Lane Auto-Negotiation Control
                '7.0001',  # Lane Auto-Negotiation Status
                '7.0010',  # Auto-Negotiation Advertisement Register 1
                '7.0011',  # Auto-Negotiation Advertisement Register 2
                '7.0012',  # Auto-Negotiation Advertisement Register 3
                '3.F000',  # Laneside Interface Mode Select Sub Port 0
                '4.F000',  # Hostside Interface Mode Select Sub Port 0
                ]
            data = ''
            for mdio_cmd in cmd_list:
                for port_nr in range(0, port_count):
                    cmd = f'2 {port_nr} {mdio_cmd}'
                    out, _, _ = bsp_comand(bsp_path, pci, 'nfp-mdio',
                                           options=cmd)
                    data = f'{data}nfp-mdio {cmd}={out}\n'
            write_logfile(data, fpath, "a")

    # Collect firmware specific logs if the interface was specified, otherwise
    # return now
    netdev = args.iface
    if netdev == '':
        return

    # Exit early if the device is not in the pci id list
    ndev_pci = netdev_to_pci(netdev)
    if ndev_pci is None or ndev_pci not in all_nfp_pcis:
        print(f"{ndev_pci} for {netdev} does not seem to be in"
              f"the detected nfp devices: {all_nfp_pcis.keys()}")
        return
    print(f'Collecting low level information for {netdev}, pci:{ndev_pci}')

    # ME_csr dumps
    MECSRS = [
        "ActCtxStatus",
        "CtxEnables",
        "CSRCtxPtr",
        "UstorErrStat",
        "ALUOut",
        "CtxArbCtrl",
        "CondCodeEn",
        "RegErrStatus",
        "LMErrStatus",
        "LMeccErrorMask",
    ]

    isls = get_enabled_islands(bsp_path, ndev_pci)
    for isl in isls:
        nr_mes = ISL_MAP[isl][1]
        if nr_mes == 0:
            continue

        fpath = f"{bspDir}/{ndev_pci}_{netdev}_MECSRS"
        data = ''
        for MECSR in MECSRS:
            me_str = f"i{isl}.me{{0..{nr_mes - 1}}}"
            cmd = f'mecsr:{me_str}.{MECSR}'
            # e.g: cmd=mecsr:i1.me{0..3}.CtxEnables
            out, _, _ = bsp_comand(bsp_path, ndev_pci, 'nfp-reg', options=cmd)
            data = f'{data}{out}\n'
        write_logfile(data, fpath, "w")

        # While in this loop also collect cls information
        fpath = f"{bspDir}/{ndev_pci}_{netdev}_cls"
        cmd = f'cls:i{isl}'
        out, _, _ = bsp_comand(bsp_path, ndev_pci, 'nfp-reg', options=cmd)
        write_logfile(out, fpath, "a")

    # Collect xpb and xpbm information.
    for xpb_cmd in ['xpb', 'xpbm']:
        # Get list of valid targets. The output needs some filtering as some
        # duplication is present
        out, _, _ = bsp_comand(bsp_path, ndev_pci, 'nfp-reg',
                               options=f"-l {xpb_cmd}")

        # Iterate through output lines, and only add to list if not already
        # present. Also filter out some of the commands that takes a long time.
        cmd_list = []
        for line in out.split():
            cmd = line.strip()
            # Skip all Nbi xpbm commands
            if "Nbi" in cmd and xpb_cmd == 'xpbm':
                continue
            # Filter Nbi xpb commands
            elif "Nbi" in cmd and xpb_cmd == 'xpb':
                cmd_list += filter_xpb_nbi(cmd)
            # Add all other commands
            elif cmd not in cmd_list:
                cmd_list.append(cmd)

        # Execute commands from the list and log to file
        for xpb in cmd_list:
            fpath = f"{bspDir}/{ndev_pci}_{netdev}_{xpb_cmd.upper()}Map"
            out, _, _ = bsp_comand(bsp_path, ndev_pci, 'nfp-reg',
                                   options=f"{xpb}")
            write_logfile(out, fpath, "a")

    # Collect rtsym information
    out, _, _ = bsp_comand(bsp_path, ndev_pci, 'nfp-rtsym', options="-L")
    rtsym_dict = parse_rtsym_table(out)

    fpath = f'{bspDir}/{ndev_pci}_{netdev}_rtsyms'
    for sym in rtsyms_to_dump:
        if sym in rtsym_dict.keys():
            write_rtsym(bsp_path, ndev_pci, rtsym_dict, sym, fpath)

def get_sosreport_cmd():
    """ Check for the existance of either 'sos' or 'sosreport'

    Return 'sos report' if tool is 'sos'
    Return 'sosreport' if tools is 'sosreport'
    Return empty string if neither was found
    """

    # Try 'sosreport'
    sos_path = shutil.which("sosreport")
    if sos_path is not None:
        return sos_path

    # Try 'sos'
    sos_path = shutil.which("sos")
    if sos_path is not None:
        return f"{sos_path} report"

    return ""

def main():
    """Call the main program functions."""
    global SCRIPT_LOG

    args = parse_cmdline()

    missing_tools = []
    for tool in REQ_SHELL_TOOLS:
        if shutil.which(tool) is None:
            missing_tools.append(tool)
    if missing_tools:
        print(f"Missing tools: {', '.join(missing_tools)}\n"
              "Please ensure these are installed on the system.")
        exit(1)

    baseDir = os.path.abspath(args.logdir)
    date, _, _ = scmd('date -u +%Y.%m.%d.%H%M')
    date = date.replace('\n', '')
    outDir = f'{baseDir}/nfp_troubleshoot_gather_{date}'
    try:
        os.makedirs(outDir, exist_ok=True)
    except PermissionError:
        print(f'Error: Seems like {baseDir} does not have write permissions. '
              'Exiting.')
        exit(1)
    SCRIPT_LOG = f'{outDir}/script_cmds.log'

    is_root = False
    if os.getuid() == 0:
        is_root = True

    if is_root is False:
        print('Warn: It is recommended to run the script with root access. '
              'Some information will not be logged without root permissions.')
        with open(SCRIPT_LOG, 'a') as logf:
            logf.write('Script NOT executed as root\n')

    common_data(outDir)
    if not sos_data(outDir):
        print("sos is not installed. Suggest install")
        print("Generate DATA without sos")
        self_data(outDir)

    # BSP tools require sudo access, only execute if running as superuser
    if is_root is True:
        bsp_path = check_bsp_tools_installed()
        if bsp_path != '' and check_cpp_access(outDir):
            collect_bsp_data(bsp_path, outDir, args)

    # Tarring up logs
    # Clear SCRIPT_LOG, since this last step cannot be logged
    # while --remove-files is used with tar
    SCRIPT_LOG = ''
    scmd(f'tar -zcvf {outDir}.tar.gz -C {baseDir} '
         f'nfp_troubleshoot_gather_{date} --remove-files')
    print(f'Done: {outDir}.tar.gz')

    # If script was run as sudo try and set permissions. Use same permissions
    # as logdir
    if is_root is True:
        stat_info = os.stat(baseDir)
        uid = stat_info.st_uid
        gid = stat_info.st_gid
        os.chown(f'{outDir}.tar.gz', uid, gid)

if __name__ == "__main__":
    main()


import asyncio
from telethon import events
import random
BILL = [
    "Yes, look at the human triangle!",
    "Remember! Reality's an illusion, the universe is a hologram, buy gold! Byeeee!",
"Pain is hilarious!",
    "I've got some children I need to make into corpses.",
    "Oh, wow, that's a great offer. How 'bout instead I shuffle the functions of every hole in your face.",
    "Sure I am insane, what's your point?",
    "Well! Well! Well!",
    "Curse you useless flesh sticks!",

]

@borg.on(events.NewMessage(pattern=r"/bill",incoming=True))
async def _(event):
    if event.fwd_from:
        return
    await event.reply(random.choice(BILL))


"""empty message

Revision ID: 69219be8b34c
Revises: 764fc66c406a
Create Date: 2019-08-10 09:09:31.692658

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '69219be8b34c'
down_revision = '764fc66c406a'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('user_groups')
    op.drop_table('groups')
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('groups',
    sa.Column('id', sa.INTEGER(), server_default=sa.text("nextval('groups_id_seq'::regclass)"), autoincrement=True, nullable=False),
    sa.Column('name', sa.VARCHAR(length=120), autoincrement=False, nullable=False),
    sa.Column('date_created', sa.DATE(), autoincrement=False, nullable=False),
    sa.PrimaryKeyConstraint('id', name='groups_pkey'),
    postgresql_ignore_search_path=False
    )
    op.create_table('user_groups',
    sa.Column('user_id', sa.INTEGER(), autoincrement=False, nullable=False),
    sa.Column('group_id', sa.INTEGER(), autoincrement=False, nullable=False),
    sa.ForeignKeyConstraint(['group_id'], ['groups.id'], name='user_groups_group_id_fkey'),
    sa.ForeignKeyConstraint(['user_id'], ['users.id'], name='user_groups_user_id_fkey'),
    sa.PrimaryKeyConstraint('user_id', 'group_id', name='user_groups_pkey')
    )
    # ### end Alembic commands ###


"""Functions for building a simplex centroid design."""

import dexpy.design as design
import pandas as pd
import numpy as np
import itertools
from dexpy.eval import count_n_choose_k as count_nk

def build_simplex_centroid(factor_count):
    """Builds a Simplex Centroid mixture design.

    This mixture design can be used for 3 to 8 components. A simplex-centroid
    design consists of all points that are equally weighted mixtures of 1 to q
    components. Included are permutations of:

        Pure blends: (1, 0, ...,0)

        Binary blends: (1/2, 1/2, 0, ...,0)

        Tertiary blends: (1/3, 1/3, 1/3, 0, ...,0)

    and so on to the overall centroid: (1/q, 1/q, ..., 1/q).

    This design differs from a simplex-lattice design. It cannot be used to
    estimate the full cubic model, but can be used to estimate a special cubic
    model.

    :param factor_count: The number of mixture components to build for.
    :type factor_count: int
    """
    run_count = 0
    for i in range(1, factor_count+1):
        run_count += count_nk(factor_count, i)

    factor_names = design.get_factor_names(factor_count)
    factor_data = pd.DataFrame(0, columns=factor_names,
                                  index=np.arange(0, run_count))

    row = 0
    for i in range(1, factor_count+1):
        proportion = 1 / i
        for combo in itertools.combinations(factor_names, i):
            factor_data.loc[row, combo] = proportion
            row += 1

    return factor_data


from django.shortcuts import render, redirect
from django.views.generic.list import ListView
from django.views.generic.edit import CreateView
from django.views.generic.detail import DetailView
from .models import Survey
from bookmark.models import Bookmark
from django.urls import reverse_lazy

# Create your views here.

class AddSurvey(CreateView):
#    return render(request, 'survey/add.html')

    model = Survey
    fields = [
        'name',
        'company',
        'mobile',
        'email',
        'position',
        'position_other',
        'q2_overall',
        'q2_sw',
        'q2_head_anc',
        'q2_xy',
        'q2_conveyor',
        'q2_feederbase',
        'q2_illumination',
        'q2_air',
        'q2_trust',
        'q2_trust_other',
        'q2_after_service',
        'q2_after_service_other',
        'q3_1_main_concern',
        'q3_21_fi_smart_erp',
        'q3_22_fi_factory_automation',
        'q3_23_fi_remote_support',
        'q3_24_fi_systematical_training',
        'q3_25_fi_other',
        'q3_25_other',
        'q3_3_recommend_to_other',
        'manager_opinion',
        'policy_agreement',
    ]
    success_url = reverse_lazy('survey_complete')
    template_name_suffix = '_create' #CreateView class template의  기본접미사(_form) 변경 : _form -> _create

def SurveyComplete(request):
    return render(request, 'survey/survey_complete.html')

class DisplaySurvey(ListView):
    model = Survey


#!/usr/bin/python

# This alternative is for users who will be creating the html file on one machine, but viewing on another.  For example, you use a remote Linux system and viewing the html is too slow and it won't fully load for printing (this is the problem Jeanette has with her computing system)

import os
import glob

# We will start with the registration png files
outfile = "/Users/jeanettemumford/Documents/Research/Talks/MumfordBrainStats/ds008_R1.1.0_raw/QA_example2/lev1_QA.html"
outdir = "/Users/jeanettemumford/Documents/Research/Talks/MumfordBrainStats/ds008_R1.1.0_raw/QA_example2/"

os.chdir(outdir)

os.system("rm %s" % (outfile))
os.system("rm -rf files")
os.system("mkdir files")

all_feats = glob.glob(
    "/Users/jeanettemumford/Documents/Research/Talks/MumfordBrainStats/ds008_R1.1.0_raw/sub*/model/task001_run00*.feat/"
)

f = open(outfile, "w")
for file in list(all_feats):
    f.write("
============================================")
    f.write("
%s" % (file))
    f.write('' % (file.replace("/", "_")))
    f.write('' % (file.replace("/", "_")))
    f.write('' % (file.replace("/", "_")))
    f.write('' % (file.replace("/", "_")))
    f.write(
        '
'
        % (file.replace("/", "_"))
    )
    f.write(
        '
'
        % (file.replace("/", "_"))
    )
    f.write(
        '
'
        % (file.replace("/", "_"))
    )
    os.system("cp %s/design.png files/%s.design.png" % (file, file.replace("/", "_")))
    os.system(
        "cp %s/design_cov.png files/%s.design_cov.png" % (file, file.replace("/", "_"))
    )
    os.system("cp %s/mc/disp.png files/%s.disp.png" % (file, file.replace("/", "_")))
    os.system("cp %s/mc/trans.png files/%s.trans.png" % (file, file.replace("/", "_")))
    os.system(
        "cp %s/reg/example_func2standard.png files/%s.example_func2standard.png"
        % (file, file.replace("/", "_"))
    )
    os.system(
        "cp %s/reg/example_func2highres.png files/%s.example_func2highres.png"
        % (file, file.replace("/", "_"))
    )
    os.system(
        "cp %s/reg/highres2standard.png files/%s.highres2standard.png"
        % (file, file.replace("/", "_"))
    )
f.close()


import numpy as np
import matplotlib.pyplot as plt

# Probability distributions of phi and Y
def sample_phi_y(size):
    return np.array([
        np.random.uniform(low=-np.pi / 2, high=np.pi / 2, size=size),
        np.random.exponential(1, size=size)
    ])

# Define random variable X
def X(phi, Y, alpha):
    return (
        np.sin(alpha * phi)
        / np.cos(phi) ** (1 / alpha)
        * (
            np.cos((1 - alpha) * phi)
            / Y
        ) ** ((1 - alpha) / alpha)
    )

if __name__ == '__main__':
    for alpha in np.linspace(1, 2, 5):
        # Random walk
        n_steps = 5000
        step_arr = X(*sample_phi_y((n_steps, 2)), alpha)
        pos_arr = np.cumsum(step_arr, axis=0)

        fig, ax = plt.subplots(tight_layout=True)
        ax.grid()
        ax.axis('equal')
        ax.set_title(f'$N = {n_steps}$, $\\alpha = {alpha}$')
        ax.set_xlabel('$x$')
        ax.set_ylabel('$y$')
        ax.plot(*pos_arr.T, 'k-')

    # PDF
    alpha = 1.7
    n_trials = 5000
    sample_X_arr = X(*sample_phi_y(n_trials), alpha)
    plt.figure()
    plt.hist(sample_X_arr, bins=100)
    plt.show()


from database.mongodb import mongodb
from kafka import KafkaConsumer
from json import loads

class consumer_problem:
    def __init__(self):
        self.database = mongodb()
        self.consumer = KafkaConsumer(
            'problemInfo',
            bootstrap_servers=['localhost:9092'],
            auto_offset_reset='earliest',
            value_deserializer=lambda x: loads(x.decode('utf-8')),
            consumer_timeout_ms=1000
        )

    def saveData(self):
        for message in self.consumer:
            data = {
                'problemId': message.value['problemId'],
                'title': message.value['titles'],
                'level': message.value['level'],
                'tag': message.value['tag']
            }
            self.database.insertProblemDocument(data)

class consumer_user:
    def __init__(self):
        self.database = mongodb()
        self.consumer = KafkaConsumer(
            'userInfo',
            bootstrap_servers=['localhost:9092'],
            auto_offset_reset='earliest',
            value_deserializer=lambda x: loads(x.decode('utf-8')),
            consumer_timeout_ms=1000
        )

    def saveData(self):
        for message in self.consumer:
            data = {
                'handle': message.value['handle'],
                'tier': message.value['tier'],
                'solvedCount': message.value['solvedCount'],
                'solvedProblems': message.value['solvedProblems']
            }
            self.database.insertUserDocument(data)


#-*- coding:utf-8 -*-
from base import *
from misc import *
import data_handler as dh
import copy
from trade_position import *
from strategy import *

class RsiAtrStrat(Strategy):
    common_params =  dict( {'price_limit_buffer': 5, 'daily_close_buffer': 2, \
                            'pos_class': 'TargetTrailTradePos', 'pos_args': {},
                            'data_func': [["RSI", "dh.RSI_F", "dh.rsi_f", {'field':'close'}], \
                                        ["ATR", "dh.ATR", "dh.atr"], \
                                        ["ATRMA", "dh.MA", "dh.ma"]]}, \
                           **Strategy.common_params)
    asset_params = dict({'rsi_win': 14, 'rsi_th': 20.0, 'atrma_win': 10, 'atr_win': 20, \
                         'freq': 1, 'stoploss': 4, }, **Strategy.asset_params)
    def __init__(self, config, agent = None):
        Strategy.__init__(self, config, agent)
        numAssets = len(self.underliers)
        self.rsi = [0.0] * numAssets
        self.atr   = [0.0] * numAssets
        self.atrma = [0.0] * numAssets
        self.tick_base = [0.0] * numAssets

    def register_func_freq(self):
        for idx, under in enumerate(self.underliers):
            for idy, infunc in enumerate(self.data_func):
                name  = infunc[0]
                sfunc = eval(infunc[1])
                rfunc = eval(infunc[2])
                if len(infunc) > 3:
                    fargs = infunc[3]
                else:
                    fargs = {}
                freq_str = str(self.freq[idx]) + 'm'
                if name  == 'ATR':
                    chan = self.atr_win[idx]
                elif name == 'ATRMA':
                    name = 'MA_ATR' + str(self.atr_win[idx]) + '_'
                    chan = self.atrma_win[idx]
                    fargs = {'field': 'ATR'+str(self.atr_win[idx])}
                elif name == "RSI":
                    chan = self.rsi_win[idx]
                fobj = BaseObject(name = name + str(chan), sfunc = fcustom(sfunc, n = chan, **fargs), rfunc = fcustom(rfunc, n = chan, **fargs))
                self.agent.register_data_func(under[0], freq_str, fobj)

    def register_bar_freq(self):
        for idx, under in enumerate(self.underliers):
            inst = under[0]
            if self.freq[idx] > 0:
                self.agent.inst2strat[inst][self.name].append(self.freq[idx])

    def initialize(self):
        self.load_state()
        for idx, underlier in enumerate(self.underliers):
            inst = underlier[0]
            self.tick_base[idx] = self.agent.instruments[inst].tick_base
            min_id = self.agent.instruments[inst].last_tick_id/1000
            min_id = int(min_id/100)*60 + min_id % 100 - self.freq[idx]
            self.last_min_id[idx] = int(min_id/60)*100 + min_id % 60
            self.update_mkt_state(idx)
        self.update_trade_unit()
        self.save_state()

    def update_mkt_state(self, idx):
        instID = self.underliers[idx][0]
        xdf = self.agent.min_data[instID][self.freq[idx]].data
        rsi_key = "RSI" + str(self.rsi_win[idx])
        self.rsi[idx] = xdf[rsi_key][-1]
        atr_key = "ATR" + str(self.atr_win[idx])
        self.atr[idx]  = xdf[atr_key][-1]
        atrma_key = "MA_ATR" + str(self.atr_win[idx]) + '_' + str(self.atrma_win[idx])
        self.atrma[idx] = xdf[atrma_key][-1]

    def on_bar(self, idx, freq):
        self.update_mkt_state(idx)
        save_status = False
        if (self.freq[idx]>0) and (freq == self.freq[idx]):
            save_status = self.check_trigger(idx)
        return save_status

    def on_tick(self, idx, ctick):
        if len(self.submitted_trades[idx]) == 0:
            curr_p = self.curr_prices[idx]
            min_id = self.agent.tick_id / 1000.0
            for tradepos in self.positions[idx]:
                if (self.close_tday[idx] and (min_id >= self.last_min_id[idx])) \
                                    or tradepos.check_exit(curr_p, self.atrma[idx] * self.stoploss[idx]):
                    msg = 'RSI-ATR to close position for inst = %s, direction=%s, current price = %s, exit target = %s' \
                                    % (tradepos.insts[0], tradepos.direction, curr_p, tradepos.exit_target)
                    self.close_tradepos(idx, tradepos, curr_p - tradepos.direction * self.tick_base[idx])
                    self.status_notifier(msg)
                    return True
                else:
                    tradepos.update_price(curr_p)
                    return False
        else:
            return False

    def check_trigger(self, idx):
        if len(self.submitted_trades[idx]) > 0:
            return False
        inst = self.underliers[idx][0]
        curr_p = self.curr_prices[idx]
        min_id = self.agent.tick_id/1000.0
        tick_base = self.tick_base[idx]
        save_status = False
        rsi_dir = (self.rsi[idx] > 50 + self.rsi_th[idx]) * 1 - (self.rsi[idx] < 50 - self.rsi_th[idx]) * 1
        open_signal = rsi_dir * (self.atr[idx] > self.atrma[idx])
        if len(self.positions[idx])>0:
            buysell = self.positions[idx][0].direction
        else:
            buysell = 0
        if rsi_dir * buysell < 0:
            tradepos = self.positions[idx][0]
            msg = 'RSI-ATR to close position for inst = %s, direction=%s, current price = %s, exit target = %s' \
                    % (tradepos.insts[0], tradepos.direction, curr_p, tradepos.exit_target)
            self.close_tradepos(idx, tradepos, curr_p - tradepos.direction * self.tick_base[idx])
            self.status_notifier(msg)
            save_status = True
        if (min_id < self.last_min_id[idx]) and (self.trade_unit[idx] > 0) \
                and (open_signal != 0) and (buysell * open_signal <= 0):
            msg = 'RSI-ATR to open position for inst = %s, ATRMA=%s, ATR=%s, RSI=%s, curr_price= %s, direction=%s, volume=%s' \
                                % (inst, self.atrma[idx], self.atr[idx], self.rsi[idx], curr_p, open_signal, self.trade_unit[idx])
            self.open_tradepos(idx, open_signal, curr_p + open_signal * tick_base)
            self.status_notifier(msg)
            save_status = True
        return save_status


from time import sleep

from stdnet import StructureFieldError
from stdnet.utils import test, populate, zip, to_string, iteritems

from examples.models import SimpleList, Dictionary, SimpleString

elems = populate('string', 100)

class MultiFieldMixin(object):
    '''Test class which add a couple of tests for multi fields.
You need to implement the get_object_and_field and adddata methods'''
    attrname = 'data'
    defaults = {}

    def setUp(self):
        self.register()

    def get_object_and_field(self, save=True, **kwargs):
        params = self.defaults.copy()
        params.update(kwargs)
        m = self.model(**params)
        if save:
            m.save()
        return m, getattr(m, self.attrname)

    def adddata(self, obj):
        raise NotImplementedError()

    def testRaiseStructFieldError(self):
        self.assertRaises(StructureFieldError, self.get_object_and_field, False)

    def testMultiFieldId(self):
        '''Here we check for multifield specific stuff like the instance
related keys (keys which are related to the instance rather than the model).'''
        # get instance and field, the field has no data here
        obj, field = self.get_object_and_field()
        # get the object id
        id = to_string(obj.id)
        # get the field database key
        field_key = to_string(field.id)
        self.assertTrue(id in field_key)
        #
        backend = obj.session.backend
        keys = backend.instance_keys(obj)
        if backend.name == 'redis':
            # field id should be in instance keys
            self.assertTrue(field.id in keys)
            lkeys = list(backend.model_keys(self.model._meta))
            # the field has no data, so there is no key in the database
            self.assertFalse(field.id in lkeys)
        #
        # Lets add data
        self.adddata(obj)
        # The field id should be in the server keys
        if backend.name == 'redis':
            lkeys = list(backend.model_keys(self.model._meta))
            self.assertTrue(field.id in lkeys)
        obj.delete()
        lkeys = list(backend.model_keys(self.model._meta))
        self.assertFalse(field.id in lkeys)

class TestStringField(MultiFieldMixin, test.CleanTestCase):
    multipledb = 'redis'
    model = SimpleString

    def adddata(self, li):
        '''Add elements to a list without using transactions.'''
        for elem in elems:
            li.data.push_back(elem)
        self.assertEqual(li.data.size(),len(''.join(elems)))

    def testIncr(self):
        m = self.model().save()
        self.assertEqual(m.data.incr(),1)
        self.assertEqual(m.data.incr(),2)
        self.assertEqual(m.data.incr(3),5)
        self.assertEqual(m.data.incr(-7),-2)

class TestListField(MultiFieldMixin, test.CleanTestCase):
    model = SimpleList
    attrname = 'names'

    def adddata(self, li):
        '''Add elements to a list without using transactions.'''
        with li.session.begin():
            names = li.names
            for elem in elems:
                names.push_back(elem)
        self.assertEqual(li.names.size(), len(elems))

    def testPushBackPopBack(self):
        li = SimpleList()
        self.assertEqual(li.id, None)
        li.save()
        names = li.names
        for elem in elems:
            names.push_back(elem)
        self.assertEqual(li.names.size(), len(elems))
        for elem in reversed(elems):
            self.assertEqual(li.names.pop_back(), elem)
        self.assertEqual(li.names.size(), 0)

    def testPushBack(self):
        li = SimpleList().save()
        with li.session.begin():
            names = li.names
            for elem in elems:
                names.push_back(elem)
        for el, ne in zip(elems, names):
            self.assertEqual(el, ne)
        self.assertEqual(li.names.size(), len(elems))

    def testPushNoSave(self):
        '''Push a new value to a list field should rise an error if the object
is not saved on databse.'''
        obj = self.model()
        push_back  = lambda : obj.names.push_back('this should fail')
        push_front = lambda : obj.names.push_front('this should also fail')
        self.assertRaises(StructureFieldError, push_back)
        self.assertRaises(StructureFieldError, push_front)

    def testPushFront(self):
        li = SimpleList().save()
        if li.session.backend.name == 'redis':
            names = li.names
            for elem in reversed(elems):
                names.push_front(elem)
            li.save()
            for el,ne in zip(elems,names):
                self.assertEqual(el,ne)

    def testPushFrontPopFront(self):
        li = SimpleList().save()
        if li.session.backend.name == 'redis':
            names = li.names
            for elem in reversed(elems):
                names.push_front(elem)
            li.save()
            self.assertEqual(li.names.size(),len(elems))
            for elem in elems:
                self.assertEqual(li.names.pop_front(),elem)
            self.assertEqual(li.names.size(),0)


class Solution:
    def lengthOfLongestSubstring(self, s: str) -> int:
        if len(s) <= 1:
            return len(s)

        longestNR = 0
        lastSeen = {}

        i = 0

        for j in range(len(s)):
            current = s[j]
            if current in lastSeen and lastSeen[current] >= i:
                i = lastSeen[current] + 1
            else:
                longestNR = max(longestNR, j - i + 1)
            lastSeen[current] = j

        return longestNR


#!/usr/bin/python3

#imports
import conf
import requests
import json
from datetime import datetime, timedelta
from dateutil import parser
import pytz
import csv

#conf
conf.load("config.json")

#constants
UP_API_ENDPOINT = "https://api.up.com.au/api/v1"
TIME_NOW = pytz.timezone(conf.get("TIMEZONE")).localize(datetime.now())
TZ = pytz.timezone(conf.get("TIMEZONE"))
#DEBUG = not __debug__
DEBUG = False

#creds
UP_CREDS_HEADER = {"Authorization": "Bearer {}".format(conf.get("UP_API_KEY"))}

def main():
    #Main function ran on script startup
    if not testUpbankApi():
        exit(10)

    bankAccounts = getUpAccounts()

    days = getDays()

    print("Getting {} days of transactions".format(days))

    for account in bankAccounts:
        saveCsv(getTransactions(account, days), account)

def getDays():
    days = int(input("Enter how many days of transactions to grab: "))
    return days

def saveCsv(transactions,account):
    output = open("data/{}.csv".format(account['attributes']['displayName']), "w", encoding="utf-8")
    writer = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC)
    writer.writerow(["Date","Payee","Memo","Amount"])
    for transaction in transactions:
        writer.writerow([
            parser.parse(transaction["attributes"]["createdAt"]).strftime('%d-%m-%Y'),
            transaction["attributes"]["description"],
            "{}: {}".format(
                parser.parse(transaction['attributes']['createdAt']).astimezone(TZ).strftime('%I:%M%p'),
                transaction['attributes']['rawText'],
            ),
            transaction["attributes"]["amount"]["value"],
        ])

def printCsv(transactions,bankAccounts):
    print("Date,Payee,Memo,Amount")
    for transaction in transactions:
        print("{},{},{},{}".format(
            parser.parse(transaction["attributes"]["createdAt"]).strftime('%d-%m-%Y'),
            transaction["attributes"]["description"],
            "Imported: " + TIME_NOW.strftime('%d-%m-%Y'),
            transaction["attributes"]["amount"]["value"],
        ))

def testUpbankApi():
    '''
    Make a call to up bank's API and parse result to determine if key is correct
    '''
    apiStatus = False

    resp = requests.get("{}/util/ping".format(UP_API_ENDPOINT),headers=UP_CREDS_HEADER)

    if resp.status_code == 200:
        if DEBUG: print("Up Bank Connected: {}".format(json.loads(resp.content)["meta"]["statusEmoji"]))
        apiStatus = True
    if resp.status_code != 200:
        print("Up Bank connection failed: GET /util/ping {}".format(resp.status_code))
        raise ValueError("Up Bank credentials may be wrong. Move the config-template.json template to config.json and replace the U API credential")

    return apiStatus

def getUpAccounts():
    #Get a list of all up bank accounts aka savers
    resp = requests.get("{}/accounts".format(UP_API_ENDPOINT),headers=UP_CREDS_HEADER)
    accounts = []

    if resp.status_code == 200:
        if DEBUG: print("Found Up Bank Accounts: ")
        for account in json.loads(resp.content)["data"]:
            if DEBUG: print("\t{}".format(account["attributes"]["displayName"]))
            accounts.append(account)
    if resp.status_code != 200:
        print("Up Bank connection failed: GET /accounts {}".format(resp.status_code))

    return accounts

def getTransactions(account, daysFrom):
    #Get past x days of transactions from the cheque account, -1 gets all
    transactions = []

    nextTransactionPage = account["relationships"]["transactions"]["links"]["related"]
    while True:
        resp = requests.get(nextTransactionPage,headers=UP_CREDS_HEADER)

        for transaction in json.loads(resp.content)["data"]:
            transactions.append(transaction)

        #If the transactions list has no more items, stop getting more transactions
        if json.loads(resp.content)["links"]["next"] is None:
            break

        #If the first transaction in the list is before a certain date, stop getting more transactions
        if daysFrom != -1:
            if datetime.fromisoformat(json.loads(resp.content)["data"][0]["attributes"]["createdAt"]) < TIME_NOW + timedelta(days=-daysFrom):
                break

        nextTransactionPage = json.loads(resp.content)["links"]["next"]

    return removeOldTransactions(transactions, daysFrom)

def removeOldTransactions(transactions, daysFrom):
    filteredTransactions = []

    if daysFrom == -1:
        filteredTransactions = transactions
    else:
        for transaction in transactions:
            if datetime.fromisoformat(transaction["attributes"]["createdAt"]) >= TIME_NOW + timedelta(days=-daysFrom):
                filteredTransactions.append(transaction)

    return filteredTransactions

def getAccountName(bankAccounts,accountId):
    for account in bankAccounts:
        if account["id"] == accountId:
            return account["attributes"]["displayName"]

if __name__ == "__main__":
    main()


#!/usr/bin/env python3
"""Classify chain-gene pairs as orthologous, paralogous etc.

Each chain-gene pair has a set of features.
We have a XGBoost pre-trained model that can classify them.
"""
import argparse
import sys
import numpy as np
import pandas as pd
import joblib
import xgboost as xgb
from version import __version__

try:  # TODO: check whether it's needed
    from modules.common import setup_logger
    from modules.common import to_log
    from modules.common import die
except ImportError:
    from modules.common import setup_logger
    from modules.common import to_log
    from common import die

__author__ = "Bogdan M. Kirilenko"

# paths to single (SE) and multi-exon (ME) models
SE_MODEL = "models/se_model.dat"
ME_MODEL = "models/me_model.dat"
LD_MODEL = "models/long_dist_model.dat"

ORTH = "ORTH"
PARA = "PARA"
SPAN = "SPAN"
P_PGENES = "P_PGENES"

SPANNING_SCORE = -1.0
PPGENE_SCORE = -2.0

# we actually extract a redundant amount of features
# lists of features required by single and multi exon models
SE_MODEL_FEATURES = ["gl_exo", "flank_cov", "exon_perc", "synt_log"]
ME_MODEL_FEATURES = ["gl_exo", "loc_exo", "flank_cov", "synt_log", "intr_perc"]
LD_MODEL_FEATURES = ["gl_exo", "flank_cov", "exon_perc", "synt_log", "loc_exo",
                     "intr_perc", "score", "single_exon"]

def parse_args():
    """Read args, check."""
    app = argparse.ArgumentParser()
    app.add_argument("table", type=str, help="Table containing merged results.")
    app.add_argument("output", type=str, help="Write the table here.")
    app.add_argument("se_model", help="A trained XGBoost model for single-exon transcripts.")
    app.add_argument("me_model", help="A trained XGBoost model for multi-exon transcripts.")
    app.add_argument(
        "--raw_model_out", "--ro", default=None, help="Save gene: chain xgboost output"
    )
    app.add_argument("--ld_model", action="store_true", dest="ld_model", help="Apply LD model")
    app.add_argument("--log_file", default=None, help="Path to the log file")
    # print help if there are no args
    if len(sys.argv) < 2:
        app.print_help()
        sys.exit(0)
    args = app.parse_args()
    return args

def classify_chains(
    table,
    output,
    se_model_path,
    me_model_path,
    raw_out=None,
    rejected=None,
    annot_threshold=0.5,
    ld_model=None
):
    """Core chain classifier function."""
    # read dataframe
    df = pd.read_csv(table, header=0, sep="\t")
    init_transcripts_set = set(df["gene"])
    to_log(f"classify_chains: loaded dataframe of size {len(df)}")
    to_log(f"classify_chains: total number of transcripts: {len(df)}")

    # get indexes of processed pseudogene chains
    # if a chain has synteny = 1, introns are deleted and exon_num > 1
    # -> then this is a proc pseudogene
    # move spanning chains to a different dataframe
    # TODO: rename trans -> spanning
    # trans chain -> a syntenic chain that passes throw the gene body
    #                but has no aligning bases in the CDS
    spanning_ts_lines = df[(df["exon_cover"] == 0) & (df["synt"] > 1)]
    to_log(f"classify_chains: {len(spanning_ts_lines)} rows with spanning chains")
    # remove from dataframe: (this includes trans chains)
    # 1) chains that don't cover CDS
    # 2) remove chains that have synteny == 0
    df = df[(df["exon_cover"] > 0) & (df["synt"] > 0)]
    df_final = df.copy()  # filtered dataframe: what we will classify
    to_log(f"classify_chains: filtered dataset contains {len(df_final)} records")
    # compute some necessary features
    to_log("classify_chains: omputing additional features...")
    df_final["exon_perc"] = df_final["exon_cover"] / df_final["ex_fract"]
    df_final["chain_len_log"] = np.log10(df_final["chain_len"])
    df_final["synt_log"] = np.log10(df_final["synt"])
    df_final["intr_perc"] = df_final["intr_cover"] / df_final["intr_fract"]
    df_final = df_final.fillna(0.0)  # fill NA values with 0.0

    if len(df_final) > 0:
        # add "is single exon" column -> to separate it for different models
        # need to do this only if the df is not empty
        df_final.loc[df_final["ex_num"] == 1, "single_exon"] = 1
        df_final.loc[df_final["ex_num"] != 1, "single_exon"] = 0
    else:  # this df is empty anyway, so any value fits
        to_log("classify_chains: WARNING! The final df for classification is empty")
        df_final["single_exon"] = 0

    # split df into two: for single and multi exon models
    df_se = df_final[df_final["single_exon"] == 1]
    df_me = df_final[df_final["single_exon"] == 0]
    to_log(f"classify_chains: df for single-exon model contains {len(df_se)} records")
    to_log(f"classify_chains: df for multi-exon model contains {len(df_me)} records")

    # create X dataframes: skip unnecessary columns
    X_se = df_se.copy()
    X_me = df_me.copy()
    X_se = X_se[SE_MODEL_FEATURES]
    X_me = X_me[ME_MODEL_FEATURES]

    # load models
    try:  # there are 2 potential problems:
        # no files at all and
        # cannot load files
        to_log(f"classify_chains: loading models at {se_model_path} (SE) and {me_model_path} (ME)")
        se_model = joblib.load(se_model_path)
        me_model = joblib.load(me_model_path)
    except FileNotFoundError:
        err_msg = (
            f"Cannot find models {se_model_path} and {me_model_path}\n"
            f"Please call train_model.py to create them.\nAbort."
        )
        to_log(f"classify_chains: ERROR! {err_msg}")
        raise FileNotFoundError(err_msg)
    except (xgb.core.XGBoostError, AttributeError):
        xgboost_version = xgb.__version__
        err_msg = (
            f"Cannot load models {se_model_path} and {me_model_path} "
            f"Probably, models were trained with a different version of "
            f"XGBoost. You used XBGoost version: {xgboost_version}; "
            f"Please make sure you called train_model.py with the same version."
        )
        to_log(f"classify_chains: ERROR! {err_msg}")
        raise ValueError(err_msg)

    # and apply them
    to_log(f"classify_chains: applying models to SE and ME datasets...")
    me_pred = me_model.predict_proba(X_me)[:, 1] if len(X_me) > 0 else np.array([])
    se_pred = se_model.predict_proba(X_se)[:, 1] if len(X_se) > 0 else np.array([])

    # add predictions to the dataframe
    # prediction is basically a single-column
    df_se_result = df_se.copy()
    df_me_result = df_me.copy()
    spanning_chains_result = spanning_ts_lines.copy()

    df_se_result["pred"] = se_pred
    df_me_result["pred"] = me_pred

    if ld_model:
        to_log(f"classify_chains: applying model for higher molecular distances")
        to_log(f"classify_chains: WARNING! This is an experimental feature")
        to_log(f"classify_chains: it is not recommended to use for research purposes yet")
        # apply LD model in addition
        # score from previous prediction is a feature for the LD model
        ld_model = joblib.load(ld_model)
        df_se_result = df_se_result.rename({"pred": "score"}, axis="columns")
        df_me_result = df_me_result.rename({"pred": "score"}, axis="columns")
        X_LD_me = df_me_result[LD_MODEL_FEATURES].copy()
        X_LD_se = df_se_result[LD_MODEL_FEATURES].copy()
        me_ld_pred = ld_model.predict_proba(X_LD_me)[:, 1] if len(X_LD_me) > 0 else np.array([])
        se_ld_pred = ld_model.predict_proba(X_LD_se)[:, 1] if len(X_LD_se) > 0 else np.array([])
        df_se_result["pred"] = se_ld_pred
        df_me_result["pred"] = me_ld_pred

    # model prediction is a float from 0 to 1, -1 -> for trans chains
    to_log(f"classify_chains: applying -1.0 score to the spanning chains")
    spanning_chains_result["pred"] = SPANNING_SCORE
    # identify processed pseudogenes, they satisfy the following criteria:
    # 1) multi-exon (single-exon ones are out of score of the method)
    # 2) synteny == 1
    # 3) cds_to_qlen > 0.95
    # set them score -2
    to_log(f"classify_chains: applying -2.0 score to the processed pseudogene alignments")
    df_me_result.loc[
        (df_me_result["synt"] == 1)
        & (df_me_result["exon_qlen"] > 0.95)
        & (df_me_result["pred"] < annot_threshold)
        & (df_me_result["exon_perc"] > 0.65),
        "pred",
    ] = PPGENE_SCORE

    pp_gene_count = df_me_result[df_me_result["pred"] == PPGENE_SCORE].shape[0]
    to_log(f"classify_chains: number of processed pseudogene alignments: {pp_gene_count}")

    # we need gene -> chain -> prediction from each row
    to_log(f"classify_chains: arranging the final output")
    df_se_result = df_se_result.loc[:, ["gene", "chain", "pred"]]
    df_me_result = df_me_result.loc[:, ["gene", "chain", "pred"]]
    spanning_chains_result = spanning_chains_result.loc[:, ["gene", "chain", "pred"]]
    # concatenate the results
    overall_result = pd.concat([df_se_result, df_me_result, spanning_chains_result])
    # some stats
    paralogs_count = overall_result[overall_result["pred"] < annot_threshold].shape[0]
    orthologs_count = overall_result[overall_result["pred"] >= annot_threshold].shape[0]
    spanning_chains_count = overall_result[overall_result["pred"] == 1.0].shape[0]
    pp_genes_count = overall_result[overall_result["pred"] == 2.0].shape[0]
    stats_msg = (
        f"* orthologs: {orthologs_count}\n"
        f"* paralogs: {paralogs_count}\n"
        f"* spanning chains: {spanning_chains_count}\n"
        f"* processed pseudogenes: {pp_genes_count}"
    )
    to_log(f"classify_chains: classification result stats:\n{stats_msg}")
    to_log(f"classify_chains: using {annot_threshold} as a threshold to separate orthologs from paralogs")

    if raw_out:  # save raw scores if required
        overall_result.to_csv(raw_out, sep="\t", index=False)

    # create a different TSV
    # transcript: lists of different chain classes
    # such as transcript A: [orthologous chains] [paralogous chains] etc
    # 0 -> placeholder, means "the class if empty"
    gene_class_chains = {}
    transcripts = set(overall_result["gene"])
    for transcript in transcripts:
        gene_class_chains[transcript] = {ORTH: [], PARA: [], SPAN: [], P_PGENES: []}

    to_log(f"classify_chains: combining results for {len(transcripts)} individual transcripts")
    for data in overall_result.itertuples():
        gene = data.gene
        chain = data.chain
        pred = data.pred

        if pred == SPANNING_SCORE:  # spanning (trans) chain
            gene_class_chains[gene][SPAN].append(chain)
        elif pred == PPGENE_SCORE:  # processed pseudogene
            gene_class_chains[gene][P_PGENES].append(chain)
        elif pred < annot_threshold:
            gene_class_chains[gene][PARA].append(chain)
        else:  # > annot_threshold
            gene_class_chains[gene][ORTH].append(chain)

    # save orthologs output
    to_log(f"classify_chains: saving the classification to {output}")
    f = open(output, "w") if output != "stdout" else sys.stdout
    f.write(f"GENE\t{ORTH}\t{PARA}\t{SPAN}\t{P_PGENES}\n")
    i = 0
    for k, v in gene_class_chains.items():
        i += 1
        orth = v[ORTH]
        para = v[PARA]
        trans = v[SPAN]
        pp = v[P_PGENES]
        orth_f = ",".join(str(x) for x in orth) if orth else "0"
        para_f = ",".join(str(x) for x in para) if para else "0"
        trans_f = ",".join(str(x) for x in trans) if trans else "0"
        p_pgenes_f = ",".join(str(x) for x in pp) if pp else "0"
        f.write("\t".join([k, orth_f, para_f, trans_f, p_pgenes_f]) + "\n")
    f.close() if output != "stdout" else None
    # for some transcripts there are no classifiable chains, save them
    transcripts_missing = list(init_transcripts_set.difference(transcripts))
    to_log(f"classify_chains: found no classifiable chains for {len(transcripts_missing)} transcripts")

    if rejected:  # we requested to save transcripts without classifiable chains
        to_log(f"classify_chains: saving these transcripts to: {rejected}")
        f = open(rejected, "w")
        for transcript in transcripts_missing:
            f.write(f"{transcript}\tNo classifiable chains\n")
        f.close()

def main():
    args = parse_args()
    setup_logger(args.log_file)
    classify_chains(
        args.table,
        args.output,
        args.se_model,
        args.me_model,
        raw_out=args.raw_model_out,
        ld_model=args.ld_model,
    )

if __name__ == "__main__":
    main()