#!/usr/local/bin/python3

import msgpack
import service
import sys
import fcntl
import os
import time

import datetime

global log_file
global log_path

if len(sys.argv) > 1:
    log_path = sys.argv[1]
else:
    log_path = "/tmp/xcbuild.diags"

log_file = None

def unpack(unpacker, data=[]):
    state = unpacker.tell()
    log("AT", state)
    try:
        obj = unpacker.unpack()
        log(obj)
    except msgpack.exceptions.OutOfData as inst:
        # Handle when the file is done
        log("Done")
        return False
    except msgpack.exceptions.StackError  as inst:
        log("StackError", state)
        log("Err", inst)
    except Exception as inst:
        log("Except-state", state)
        log("At", data[state:state+400])
        log("Err", inst)
        try:
            unpacker.skip()
        except:
            pass
    return True

def log(*args, **kwargs):
    global log_file
    global log_path

    now = datetime.datetime.now()
    if log_file == None:
        log_file = open(log_path,"w", 512)
    log_file.write(str(now) + " INFO: "+" ".join(map(str,args))+"\n", **kwargs)

def dump_protocol(path, sanitize=False):
    global log_path
    log_path = path + ".diags"
    data_file = open(path, 'rb')
    data = data_file.read()
    log("Data:", data)
    unpacker = service.get_unpacker()
    unpacker.feed(data)
    if sanitize:
        data = service.sanitize(data)

    while True:
        if not unpack(unpacker, data):
            return

def loop():
    global log_file
    log("Start")
    unpacker = service.get_unpacker()
    buff = None
    last_byte = None
    byte = None
    i = 0
    while True:
        last_byte = byte
        byte = sys.stdin.buffer.read(1)
        log_file.flush()
        if not byte:
            i = i + 1
            if i > 10:
                log("Waiting")
                time.sleep(2.0)
        else:
            i = 0
        if not buff:
            buff = byte
        if byte:
            buff += byte
        if (not byte or byte == b'') and buff:
            unpacker.feed(buff)
            while unpack(unpacker, buff):
                log("Pack")
            buff = None
            byte = None
            unpacker = service.get_unpacker()

orig_fl = fcntl.fcntl(sys.stdin, fcntl.F_GETFL)
fcntl.fcntl(sys.stdin, fcntl.F_SETFL, orig_fl | os.O_NONBLOCK)
loop()


from bs4 import BeautifulSoup
import requests
# with open("website.html") as website:
#     contents = website.read()

# soup = BeautifulSoup(contents, "html.parser")

# anchors = soup.find_all(name="a")

# print(anchors)

response = requests.get("https://news.ycombinator.com/news").text

# print(website)

soup = BeautifulSoup(response, "html.parser")

articleText = soup.find(name="a", class_="storylink").getText()

articleLink = soup.find(name="a", class_="storylink").get("href")

articleUpvote = (soup.find(name="span", class_="score").getText())

print(articleText)
print(articleText)
print(articleUpvote)


#!/usr/bin/env python

import numpy as np
from skimage import io, segmentation
from scipy.ndimage import gaussian_filter

import rclpy
import tf2_ros
import tf_transformations
import tf2_geometry_msgs
import tf2_py
from rclpy.node import Node
from nav_msgs.msg import OccupancyGrid
from nav_msgs.srv import GetMap
from sensor_msgs.msg import LaserScan

def yaw_from_orientation(orientation):
    return tf_transformations.euler_from_quaternion((orientation.x, orientation.y, orientation.z, orientation.w))[2]

class InflationNode(Node):
    def __init__(self):
        super().__init__('inflation_node')
        self.get_logger().info('Starting')
        self.robot_frames = ['robot1', 'robot2']
        self.get_logger().info(self.get_namespace())
        self.robot_frames.remove(self.get_namespace().split('/')[1])
        self.own_frame = 'base_link'
        self.reference_frame = 'map'

        self.safety_dist = 0.15

        self.tfBuffer = tf2_ros.Buffer(cache_time=rclpy.time.Duration(seconds=3.0))
        self.tfListener = tf2_ros.TransformListener(self.tfBuffer, self)

        self.raw_map = None
        # create a qos profile to receive the last sent map
        qos_profile = rclpy.qos.qos_profile_system_default
        qos_profile.reliability = rclpy.qos.QoSReliabilityPolicy.RMW_QOS_POLICY_RELIABILITY_RELIABLE
        qos_profile.durability = rclpy.qos.QoSDurabilityPolicy.RMW_QOS_POLICY_DURABILITY_TRANSIENT_LOCAL
        self.create_subscription(OccupancyGrid, 'map', self.map_cb, qos_profile)
        self.map_publisher = self.create_publisher(OccupancyGrid, 'nav/obstacles', 10)
        self.scan = None
        self.create_subscription(LaserScan, 'scan', self.scan_cb, rclpy.qos.qos_profile_sensor_data)
        self.create_timer(0.5, self.timer_cb)
        self.map_client = self.create_client(GetMap, 'map_server/map')
        self.map_client.wait_for_service()
        future = self.map_client.call_async(GetMap.Request())
        rclpy.spin_until_future_complete(self, future)
        self.update_map(future.result().map)
        self.get_logger().info('init done')

    def scan_cb(self, msg):
        self.scan = msg

    def scan_image(self):
        try:
            trans = self.tfBuffer.lookup_transform(self.raw_map.header.frame_id, self.scan.header.frame_id, rclpy.time.Time().to_msg())
        except Exception as e:
            self.get_logger().warn(f'{e}')
            return np.zeros_like(self.map_image)

        scan_img = np.zeros_like(self.map_image)

        own_theta = yaw_from_orientation(trans.transform.rotation)
        r_max = 2.0

        for i, r in enumerate(self.scan.ranges):
            if r > r_max:
                continue
            x = trans.transform.translation.y + r * np.sin(own_theta + i / 180.0 * np.pi)
            y = trans.transform.translation.x + r * np.cos(own_theta + i / 180.0 * np.pi)
            x, y = self.metric_to_px_coorditaes(x, y)
            if x < 0 or x >= self.raw_map.info.width:
                continue
            if y < 0 or y >= self.raw_map.info.height:
                continue
            scan_img[x, y] = 100
        return scan_img

    def convert_map_data_to_image(self, map_data):
        image = np.array(map_data.data, dtype=int)
        image = np.reshape(image, (map_data.info.width, map_data.info.height))
        image[image < 50] = 0
        image[image >= 50] = 100
        return image

    def metric_to_px_coorditaes(self, x, y):
        x = (x - self.raw_map.info.origin.position.x ) / self.raw_map.info.resolution
        y = (y - self.raw_map.info.origin.position.y ) / self.raw_map.info.resolution
        return int(x), int(y)

    def transform_to_px_coordinates(self, t):
        y = (t.transform.translation.x - self.raw_map.info.origin.position.x) / self.raw_map.info.resolution
        x = (t.transform.translation.y - self.raw_map.info.origin.position.y) / self.raw_map.info.resolution
        return int(x), int(y)

    def other_robot_image(self):
        other = np.zeros_like(self.map_image)
        for _, robot_frame in enumerate(self.robot_frames):
            try:
                trans = self.tfBuffer.lookup_transform(self.raw_map.header.frame_id, robot_frame, rclpy.time.Time().to_msg())
                x, y = self.transform_to_px_coordinates(trans)
                #self.get_logger().info(f'{robot_frame}: {x}, {y}')

                other[x][y] = 100
                r = self.safety_dist * 2
                theta = yaw_from_orientation(trans.transform.rotation)
                x = trans.transform.translation.y + r * np.sin(theta)
                y = trans.transform.translation.x + r * np.cos(theta)
                x, y = self.metric_to_px_coorditaes(x, y)
                if x < 0 or x >= self.raw_map.info.width:
                    continue
                if y < 0 or y >= self.raw_map.info.height:
                    continue
                other[x, y] = 100
            except Exception as e:
                self.get_logger().warn(f'{e}')
        return other

    def combine_sensor_sources(self):
        other = segmentation.expand_labels(self.other_robot_image(), distance=2.0 * self.safety_dist/self.raw_map.info.resolution)
        image = segmentation.expand_labels(self.map_image, distance=self.safety_dist/self.raw_map.info.resolution)
        scan_image = segmentation.expand_labels(self.scan_image(), distance=self.safety_dist/self.raw_map.info.resolution)
        image = gaussian_filter(image + other + scan_image, 1)
        image[image > 100] = 100
        image[image <= 0] = 0
        return image

    def convert_image_to_map_data(self, image):
        map_data = self.raw_map
        map_data.data = [int(x) for x in image.flat]
        return map_data

    def update_map(self, map_msg):
        self.raw_map = map_msg
        #extract map as image
        self.map_image = self.convert_map_data_to_image(map_msg)

    def map_cb(self, msg):
       self.get_logger().info('received a map callback\n \n \n xxxxxxxxxxxxxxxxxxxxxxxxxx \n \n \n')
       self.update_map(msg)

    def timer_cb(self):
        if self.raw_map is not None:
            self.map_publisher.publish(self.convert_image_to_map_data(self.combine_sensor_sources()))
        else:
            self.get_logger().info('no map received yet')

def main():
    rclpy.init()
    node = InflationNode()
    try:
        rclpy.spin(node)
    except KeyboardInterrupt:
        node.get_logger().info('Shutting Down')
        node.destroy_node()

if __name__ == "__main__":
    main()


import os
import json
import torch
import numpy as np
from requests import post
from argparse import ArgumentParser
from tqdm import tqdm
from opennre import encoder
from opennre.model import SoftmaxNN
from functools import partial
import logging

from config import *
from server.saliency_utils import *

# Set global device
if DEVICE == 'gpu':
    device = torch.device(
        'cuda:0') if torch.cuda.is_available() else torch.device('cpu')
else:
    device = torch.device('cpu')

# Storages for hooks to fetch gradient & input
input_storage = []
gradient_storage = []

# Load model & register hooks
def load_model(_dataset_name, _model_name, _rel2id):
    # Silent unimportant log messages
    for logger_name in ['transformers.configuration_utils',
                        'transformers.modeling_utils',
                        'transformers.tokenization_utils_base', 'absl']:
        logging.getLogger(logger_name).setLevel(logging.WARNING)
    # Load model by path
    _encoders = {'bert': encoder.BERTEncoder,
                 'bertentity': encoder.BERTEntityEncoder}
    if _model_name in _encoders:
        _sentence_encoder = _encoders[_model_name](
            max_length=MAX_SEQ_LEN,
            pretrain_path=PRETRAIN_PATH,
            mask_entity=False
        )
        _model = SoftmaxNN(_sentence_encoder, len(_rel2id), _rel2id)
        _model.load_state_dict(torch.load(MODEL_PATH_DICT[_dataset_name][_model_name],
                                          map_location=device)['state_dict'])
        # input_data is an empty tuple
        # output_data is tensor (batch, seq_len, model_dim)
        _model.sentence_encoder.bert.embeddings.register_forward_hook(
            lambda module, input_data, output_data:
            input_storage.append(
                output_data.detach().squeeze(0).cpu().numpy()))  # (128, 768)
        # grad_input is a tuple containing one tensor (batch, seq_len, model_dim)
        # grad_output is a tuple containing one tensor (batch, seq_len, model_dim)
        _model.sentence_encoder.bert.embeddings.register_backward_hook(
            lambda module, grad_input, grad_output:
            gradient_storage.append(
                grad_input[0].detach().squeeze(0).cpu().numpy()))  # (128, 768)
        _model.to(device)
        _model.eval()

        return _model
    else:
        raise NotImplementedError

def get_saliency_local(model, sample, rel2id, id2rel):
    def get_saliency(unsupervised=False):
        def forward_backward(_inputs):
            _logits = sentence_encoder(*_inputs)
            _logits = model.softmax(model.fc(model.drop(_logits)))
            if unsupervised:
                # Use model's prediction as target
                _, pred = _logits.max(-1)
                target_label_id = pred[0].item()
            else:
                # Use supervised label as target
                target_label_id = rel2id[sample['relation']]
            ce = torch.nn.CrossEntropyLoss()
            loss = ce(_logits, torch.LongTensor([target_label_id]).to(device))
            loss.backward()
            return _logits[0].detach().cpu().numpy()

        # Tokenize sample
        sentence_encoder = model.sentence_encoder
        inputs = sentence_encoder.tokenize(sample)
        inputs = [i.to(device) for i in inputs]
        token_ids = inputs[0][0]
        tokens = sentence_encoder.tokenizer.convert_ids_to_tokens(token_ids)
        if '[PAD]' in tokens:
            tokens = tokens[:tokens.index('[PAD]')]

        # Process forward-backward once
        logits = forward_backward(inputs)
        pred = logits.argmax()
        confidence = float(logits[pred])
        pred_label = id2rel[pred]

        # Start calculating integrated gradients
        alpha_levels = 10
        for alpha_i in range(1, alpha_levels):
            alpha_rate = 1.0 * alpha_i / alpha_levels
            # Register new hook to modify embedding output each time
            alpha_hook = sentence_encoder.bert.embeddings.register_forward_hook(
                lambda module, input_data, output_data: output_data * alpha_rate)

            try:
                _ = forward_backward(inputs)
            except Exception as e:
                alpha_hook.remove()
                raise e

            alpha_hook.remove()
        gradient_average = np.mean(gradient_storage, axis=0)
        # Use first time's input
        saliency = gradient_average * input_storage[0]

        # Convert float32 to float, so that the dict is JSON serializable
        saliency = normalize(saliency[:len(tokens)])
        saliency = [float(score) for score in saliency]
        token_saliency = list(zip(tokens, saliency))
        word_saliency = merge_token_saliency(token_saliency)

        # Clear input / gradient storage
        input_storage.clear()
        gradient_storage.clear()

        return {'pred_label': pred_label,
                'confidence': confidence,
                'word_saliency': word_saliency}

    label = sample['relation']
    saliency_dict = {'label': label,
                     'prediction': '',
                     'confidence': 0.0,
                     'supervised': None,
                     'unsupervised': None}
    supervised_dict = get_saliency()
    saliency_dict['prediction'] = supervised_dict['pred_label']
    saliency_dict['confidence'] = supervised_dict['confidence']
    saliency_dict['supervised'] = supervised_dict['word_saliency']
    if supervised_dict['pred_label'] != label:
        unsupervised_dict = get_saliency(unsupervised=True)
        saliency_dict['unsupervised'] = unsupervised_dict['word_saliency']

    return saliency_dict

def get_saliency_server(args, sample):
    url = 'http://localhost:{}/saliency'.format(PORT)
    label = sample['relation']
    model = args.model
    dataset = args.dataset
    method = 'integrated_gradient'
    saliency_dict = {'label': label,
                     'prediction': '',
                     'confidence': 0.0,
                     'supervised': None,
                     'unsupervised': None}
    r = post(url, data=json.dumps({'model': model,
                                   'dataset': dataset,
                                   'method': method,
                                   'sample': sample}))
    supervised_dict = json.loads(r.text)
    saliency_dict['prediction'] = supervised_dict['pred_label']
    saliency_dict['confidence'] = supervised_dict['confidence']
    saliency_dict['supervised'] = supervised_dict['word_saliency']
    if supervised_dict['pred_label'] != label:
        r = post(url, data=json.dumps({'model': model,
                                       'dataset': dataset,
                                       'method': method,
                                       'sample': sample,
                                       'unsupervised': True}))
        unsupervised_dict = json.loads(r.text)
        saliency_dict['unsupervised'] = unsupervised_dict['word_saliency']

    return saliency_dict

if __name__ == "__main__":
    parser = ArgumentParser('Saliency fetching tool')
    parser.add_argument('--dataset', '-d', required=True,
                        choices=['tacred', 'wiki80'])
    parser.add_argument('--model', '-m', required=True,
                        choices=['bert', 'bertentity'])
    parser.add_argument('--adversary', '-a', default='none',
                        choices=['none', 'pw', 'tf', 'hf', 'uat', 'gen'])
    parser.add_argument('--mode', default='local', choices=['local', 'server'])

    args = parser.parse_args()

    if args.mode == 'local':
        rel2id = json.load(open(RELATION_PATH_DICT[args.dataset], 'r'))
        id2rel = {v: k for (k, v) in rel2id.items()}
        model = load_model(args.dataset, args.model, rel2id)
        saliency_func = partial(
            get_saliency_local, model=model, rel2id=rel2id, id2rel=id2rel)
    else:
        saliency_func = partial(get_saliency_server, args=args)

    if args.adversary == 'none':
        for data_type in DATA_TYPES[args.dataset]:
            input_file = os.path.join(
                DATA_ROOT, args.dataset, data_type + '.txt')
            output_file = os.path.join(
                DATA_ROOT, args.dataset, args.model, data_type + '_ig.txt')
            if os.path.exists(output_file):
                progress = len(open(output_file, 'r').readlines())
                logging.info('Restore progress from {} at index {}'.format(
                    output_file, progress))
            else:
                progress = 0
            out_handler = open(output_file, 'a')
            samples = [json.loads(line)
                       for line in open(input_file, 'r').readlines()]
            for sample in tqdm(samples[progress:],
                               desc='Fetching saliency for samples in ' + input_file):
                saliency = saliency_func(sample=sample)
                out_handler.write(json.dumps(saliency) + '\n')
    else:
        for data_type in DATA_TYPES[args.dataset]:
            input_file = os.path.join(
                DATA_ROOT, args.dataset, args.model,
                '{}_{}.txt'.format(data_type, args.adversary))
            output_file = os.path.join(
                DATA_ROOT, args.dataset, args.model,
                '{}_{}_ig.txt'.format(data_type, args.adversary))
            if os.path.exists(output_file):
                progress = max([json.loads(line)['index']
                                for line in open(output_file, 'r').readlines()])
                logging.info('Restore progress from {} at {}'.format(
                    output_file, progress))
            else:
                progress = -1
            out_handler = open(output_file, 'a')
            samples = [json.loads(line)
                       for line in open(input_file, 'r').readlines()]
            samples = [s for s in samples if s['index'] > progress]
            for sample_dict in tqdm(samples, desc='Fetching saliency for samples in ' + input_file):
                index = sample_dict['index']
                for sample in sample_dict['adversary_samples']:
                    saliency = saliency_func(sample=sample)
                    saliency['index'] = index
                    out_handler.write(json.dumps(saliency) + '\n')


from datetime import datetime, timedelta
from unittest.mock import patch

import pandas as pd

from src.pipeline.shared_tasks.alerts import (
    archive_reporting,
    extract_non_archived_reportings_ids_of_type,
    extract_pending_alerts_ids_of_type,
    extract_silenced_alerts,
    filter_silenced_alerts,
    load_alerts,
    make_alerts,
    validate_pending_alert,
)
from src.read_query import read_query
from tests.mocks import mock_datetime_utcnow

def test_extract_silenced_alerts(reset_test_data):
    silenced_alerts = extract_silenced_alerts.run()
    expected_silenced_alerts = pd.DataFrame(
        {
            "internal_reference_number": ["ABC000658985", "ABC000542519"],
            "external_reference_number": ["OHMYGOSH", "RO237719"],
            "ircs": ["OGMJ", "FQ7058"],
            "type": ["THREE_MILES_TRAWLING_ALERT", "MISSING_FAR_ALERT"],
        }
    )
    pd.testing.assert_frame_equal(silenced_alerts, expected_silenced_alerts)

def test_extract_pending_alerts_ids_of_type(reset_test_data):
    assert extract_pending_alerts_ids_of_type.run(
        alert_type="THREE_MILES_TRAWLING_ALERT"
    ) == [12]

    assert extract_pending_alerts_ids_of_type.run(alert_type="NON_EXISTING_ALERT") == []

def test_extract_non_archived_reportings_ids_of_type(reset_test_data):
    assert extract_non_archived_reportings_ids_of_type.run(
        reporting_type="THREE_MILES_TRAWLING_ALERT"
    ) == [56]

    assert (
        extract_non_archived_reportings_ids_of_type.run(
            reporting_type="NON_EXISTING_ALERT"
        )
        == []
    )

@patch("src.pipeline.shared_tasks.alerts.requests")
def test_validate_pending_alert(requests_mock):
    validate_pending_alert.run(12)
    requests_mock.put.assert_called_once_with(
        "https://monitor.fish/api/v1/operational_alerts/12/validate"
    )

@patch("src.pipeline.shared_tasks.alerts.requests")
def test_archive_reporting(requests_mock):
    archive_reporting.run(12)
    requests_mock.put.assert_called_once_with(
        "https://monitor.fish/api/v1/reportings/12/archive"
    )

@patch(
    "src.pipeline.shared_tasks.alerts.datetime",
    mock_datetime_utcnow(datetime(2020, 5, 3, 8, 0, 0)),
)
def test_make_alerts():
    date_1 = datetime(2020, 1, 2, 11, 12, 30)
    date_2 = datetime(2022, 1, 2, 10, 10, 0)
    # With creation_date in input
    vessels_in_alert = pd.DataFrame(
        {
            "cfr": ["A", "B"],
            "external_immatriculation": ["AA", "BB"],
            "ircs": ["AAA", "BBB"],
            "flag_state": ["FR", "BE"],
            "vessel_name": ["Vessel_A", "Vessel_B"],
            "facade": ["NAMO", "MEMN"],
            "vessel_identifier": [
                "INTERNAL_REFERENCE_NUMBER",
                "INTERNAL_REFERENCE_NUMBER",
            ],
            "risk_factor": [1.23, 3.56],
            "creation_date": [date_1, date_2],
            "latitude": [9.8, -1.963],
            "longitude": [65.59, -81.71],
            "vessel_id": [1, 12],
            "dml": ["dml 007", "dml 22"],
        }
    )

    alerts = make_alerts.run(
        vessels_in_alert,
        alert_type="MISSING_FAR_ALERT",
        alert_config_name="MISSING_FAR_ALERT_CONFIG_1",
    )

    expected_alerts = pd.DataFrame(
        {
            "vessel_name": ["Vessel_A", "Vessel_B"],
            "internal_reference_number": ["A", "B"],
            "external_reference_number": ["AA", "BB"],
            "ircs": ["AAA", "BBB"],
            "flag_state": ["FR", "BE"],
            "vessel_id": [1, 12],
            "vessel_identifier": [
                "INTERNAL_REFERENCE_NUMBER",
                "INTERNAL_REFERENCE_NUMBER",
            ],
            "creation_date": [date_1, date_2],
            "latitude": [9.8, -1.963],
            "longitude": [65.59, -81.71],
            "type": ["MISSING_FAR_ALERT", "MISSING_FAR_ALERT"],
            "value": [
                {
                    "seaFront": "NAMO",
                    "type": "MISSING_FAR_ALERT",
                    "riskFactor": 1.23,
                    "dml": "dml 007",
                },
                {
                    "seaFront": "MEMN",
                    "type": "MISSING_FAR_ALERT",
                    "riskFactor": 3.56,
                    "dml": "dml 22",
                },
            ],
            "alert_config_name": [
                "MISSING_FAR_ALERT_CONFIG_1",
                "MISSING_FAR_ALERT_CONFIG_1",
            ],
        }
    ).astype({"creation_date": "datetime64[ns]"})

    pd.testing.assert_frame_equal(alerts, expected_alerts)

    # Without optional in input
    vessels_in_alert = vessels_in_alert.drop(
        columns=["creation_date", "latitude", "longitude"]
    )
    alerts = make_alerts.run(
        vessels_in_alert,
        alert_type="MISSING_FAR_ALERT",
        alert_config_name="MISSING_FAR_ALERT_CONFIG_1",
    )
    expected_alerts["latitude"] = None
    expected_alerts["longitude"] = None
    expected_alerts["creation_date"] = [
        datetime(2020, 5, 3, 8, 0, 0),
        datetime(2020, 5, 3, 8, 0, 0),
    ]
    expected_alerts = expected_alerts.astype({"creation_date": "datetime64[us]"})

    pd.testing.assert_frame_equal(alerts, expected_alerts)

def test_filter_silenced_alerts():
    now = datetime(2020, 1, 1, 0, 0, 0)
    td = timedelta(hours=1)
    alert_type = "USER_DEFINED_ALERT_TYPE"
    alert_config_name = "ALERTE_CHALUTAGE_CONFIG_1"

    alerts = pd.DataFrame(
        {
            "vessel_name": ["v_A", "v_B", "v_C"],
            "internal_reference_number": ["A", "B", "C"],
            "external_reference_number": ["AA", "BB", "CC"],
            "ircs": ["AAA", "BBB", "CCC"],
            "flag_state": ["FR", "FR", "FR"],
            "vessel_id": [1, 12, 15],
            "vessel_identifier": [
                "INTERNAL_REFERENCE_NUMBER",
                "INTERNAL_REFERENCE_NUMBER",
                "INTERNAL_REFERENCE_NUMBER",
            ],
            "creation_date": [now, now - 0.5 * td, now - td],
            "latitude": [9.8, -1.963, -2.365],
            "longitude": [65.59, -81.71, 46.894],
            "type": [alert_type, alert_type, alert_type],
            "value": [
                {
                    "seaFront": "NAMO",
                    "type": alert_type,
                    "riskFactor": 1.23,
                    "dml": "dml A",
                },
                {
                    "seaFront": "MEMN",
                    "type": alert_type,
                    "riskFactor": None,
                    "dml": "dml B",
                },
                {
                    "seaFront": "MEMN",
                    "type": alert_type,
                    "riskFactor": 2.56,
                    "dml": "dml C",
                },
            ],
            "alert_config_name": [
                alert_config_name,
                alert_config_name,
                alert_config_name,
            ],
        }
    )

    silenced_alerts = pd.DataFrame(
        {
            "internal_reference_number": ["A", "B_ANOTHER_VESSEL", "C"],
            "external_reference_number": ["AA", "BB_ANOTHER_VESSEL", "CC"],
            "ircs": ["AAA", "BBB", "CCC"],
            "type": [alert_type, alert_type, "ANOTHER_ALERT_TYPE"],
        }
    )

    active_alerts = filter_silenced_alerts.run(alerts, silenced_alerts)

    expected_active_alerts = pd.DataFrame(
        {
            "vessel_name": ["v_B", "v_C"],
            "internal_reference_number": ["B", "C"],
            "external_reference_number": ["BB", "CC"],
            "ircs": ["BBB", "CCC"],
            "flag_state": ["FR", "FR"],
            "vessel_id": [12, 15],
            "vessel_identifier": [
                "INTERNAL_REFERENCE_NUMBER",
                "INTERNAL_REFERENCE_NUMBER",
            ],
            "creation_date": [now - 0.5 * td, now - td],
            "latitude": [-1.963, -2.365],
            "longitude": [-81.71, 46.894],
            "value": [
                {
                    "seaFront": "MEMN",
                    "type": alert_type,
                    "riskFactor": None,
                    "dml": "dml B",
                },
                {
                    "seaFront": "MEMN",
                    "type": alert_type,
                    "riskFactor": 2.56,
                    "dml": "dml C",
                },
            ],
            "alert_config_name": [alert_config_name, alert_config_name],
        }
    ).reset_index(drop=True)

    pd.testing.assert_frame_equal(
        active_alerts.reset_index(drop=True), expected_active_alerts
    )

def test_load_alerts(reset_test_data):
    initial_alerts = read_query("SELECT * FROM pending_alerts", db="monitorfish_remote")

    now = datetime(2020, 1, 1, 0, 0, 0)
    td = timedelta(hours=1)
    alert_type = "FRENCH_EEZ_FISHING_ALERT"
    alert_config_name = "ALERTE_CHALUTAGE_CONFIG_1"
    alerts_to_load = pd.DataFrame(
        {
            "vessel_name": ["v_B"],
            "internal_reference_number": ["B"],
            "external_reference_number": ["BB"],
            "ircs": ["BBB"],
            "flag_state": ["FR"],
            "vessel_id": [12],
            "vessel_identifier": [
                "INTERNAL_REFERENCE_NUMBER",
            ],
            "creation_date": [now - 0.5 * td],
            "latitude": [-1.963],
            "longitude": [-81.71],
            "value": [
                {
                    "seaFront": "MEMN",
                    "type": alert_type,
                    "riskFactor": None,
                    "dml": "dml B",
                },
            ],
            "alert_config_name": [alert_config_name],
        }
    )

    load_alerts.run(alerts_to_load, alert_config_name=alert_config_name)
    final_alerts = read_query("SELECT * FROM pending_alerts", db="monitorfish_remote")
    assert len(final_alerts) == len(initial_alerts) + 1


import os

import spiceypy
from spiceypy.utils.exceptions import NotFoundError
from spiceypy.utils.support_types import SpiceyError

from spival.utils.skd_constants import *
from spival.utils.files import is_empty_file, \
    has_badchars, \
    exceeds_line_lengths, \
    commnt_read, \
    get_section_text_from_kernel_comments, \
    get_kernel_version, \
    validate_indentation, \
    validate_trailing_chars, read_all_text, get_text_and_data_from_kernel, get_sections_map_from_kernel_comments, \
    get_section_from_sections_map, get_naif_ids_from_text, \
    get_frames_definitions_from_text, get_instruments_definitions_from_text, get_sites_definitions_from_text

# Modification of:
# https://spiceypy.readthedocs.io/en/main/other_stuff.html#lesson-1-kernel-management-with-the-kernel-subsystem
from spival.utils.skd_val_logger import log_error, log_info, log_warn

FOUND_NAIF_IDS = []
FOUND_FRAME_IDS = []
FOUND_FRAME_NAMES = []

def write_mk_kernels_report(mk_path, out_report_file):

    # Prepare report file for writing
    out_file = open(out_report_file, 'w+')

    try:
        # Load the meta kernel then use KTOTAL to interrogate the SPICE
        # kernel subsystem.
        out_file.write('Loading MK: {0}\n'.format(mk_path))
        spiceypy.furnsh(mk_path)

        count = spiceypy.ktotal('ALL')
        out_file.write('Kernel count after load: {0}\n'.format(count))

        # Loop over the number of files; interrogate the SPICE system
        # with spiceypy.kdata for the kernel names and the type.
        # 'found' returns a boolean indicating whether any kernel files
        # of the specified type were loaded by the kernel subsystem.
        # This example ignores checking 'found' as kernels are known
        # to be loaded.
        for i in range(0, count):
            [file, ktype, source, handle] = spiceypy.kdata(i, 'ALL')
            out_file.write('File   {0}\n'.format(file))
            out_file.write('Type   {0}\n'.format(ktype))
            out_file.write('Source {0}\n\n'.format(source))

        # Lets write the first 1000 variables and their values
        cvals = spiceypy.gnpool("*", 0, 1000)
        for cval in cvals:

            #
            # Use spiceypy.dtpool to return the dimension and type,
            # C (character) or N (numeric), of each pool
            # variable name in the cvals array. We know the
            # kernel data exists.
            #
            [dim, type] = spiceypy.dtpool(cval)

            out_file.write('\n' + cval)
            out_file.write(' Number items: {0}   Of type: {1}\n'.format(dim, type))

            #
            # Test character equality, 'N' or 'C'.
            #
            if type == 'N':

                #
                # If 'type' equals 'N', we found a numeric array.
                # In this case any numeric array will be an array
                # of double precision numbers ('doubles').
                # spiceypy.gdpool retrieves doubles from the
                # kernel pool.
                #
                dvars = spiceypy.gdpool(cval, 0, 10)
                for dvar in dvars:
                    out_file.write('  Numeric value: {0:20.6f}\n'.format(dvar))

            elif type == 'C':

                #
                # If 'type' equals 'C', we found a string array.
                # spiceypy.gcpool retrieves string values from the
                # kernel pool.
                #
                cvars = spiceypy.gcpool(cval, 0, 10)

                for cvar in cvars:
                    out_file.write('  String value: {0}\n'.format(cvar))

            else:

                #
                # This block should never execute.
                #
                out_file.write('Unknown type. Code error.\n')

        # Now unload the meta kernel. This action unloads all
        # files listed in the meta kernel.
        spiceypy.unload(mk_path)

    except SpiceyError as err:
        out_file.write(str(err))

    # Done. Unload the kernels.
    spiceypy.kclear()

    return

def is_valid_kernel(kernel_file, check_line_length=True, check_indentation=True, check_trailing_chars=True):
    is_valid = True

    extension = str(os.path.splitext(kernel_file)[1]).lower()
    if extension in KERNEL_EXTENSIONS:

        if extension in KERNEL_TEXT_EXTENSIONS:
            if not is_valid_text_kernel(kernel_file,
                                        check_line_length, check_indentation, check_trailing_chars):
                log_error("INVALID_TEXT_KERNEL", "Invalid text kernel.", kernel_file)
                is_valid = False

        else:
            # Is a binary kernel file, extract comments:
            if not is_valid_binary_kernel(kernel_file,
                                          check_line_length, check_indentation, check_trailing_chars):
                log_error("INVALID_BINARY_KERNEL", "Invalid binary kernel.", kernel_file)
                is_valid = False

        if is_mk_file(kernel_file):
            if not is_valid_metakernel(kernel_file):
                log_error("INVALID_METAKERNEL", "Invalid meta-kernel.", kernel_file)
                is_valid = False

    else:
        log_error("WRONG_KERNEL_EXTENSION", "Invalid kernel extension: " + extension, kernel_file)
        is_valid = False

    return is_valid

def is_valid_text_kernel(filename, check_line_length=True, check_indentation=True, check_trailing_chars=True):
    is_valid = True

    if not has_valid_text_kernel_header(filename):
        is_valid = False

    if not is_valid_comment_file(filename, check_line_length, check_indentation, check_trailing_chars):
        is_valid = False

    if not has_valid_version_and_date_section(filename):
        is_valid = False

    if not has_valid_contact_section(filename):
        is_valid = False

    if is_fk_file(filename):
        if not is_valid_frames_kernel(filename):
            log_error("INVALID_FRAMES_KERNEL", "Invalid frames kernel.", filename)
            is_valid = False

    elif is_ik_file(filename):
        if not is_valid_instruments_kernel(filename):
            log_error("INVALID_INSTRUMENTS_KERNEL", "Invalid instruments kernel.", filename)
            is_valid = False

    # TODO: Do specific checks for text PCKs and SCLKs

    return is_valid

def is_valid_comment_file(filename, check_line_length=True, check_indentation=True, check_trailing_chars=True):

    if is_empty_file(filename):
        log_error("EMPTY_FILE", "File comments are empty", filename)
        return False

    is_valid = True
    if has_badchars(filename):
        log_error("HAS_BAD_CHARS", "Has bad chars.", filename)
        is_valid = False

    if check_line_length:
        if exceeds_line_lengths(filename, ignore_lines=COMMENTS_FILE_IGNORE_LINES):
            log_error("HAS_LONG_LINES", "Has long lines.", filename)
            is_valid = False

    if check_indentation:
        if validate_indentation(filename, indentation=3):
            log_error("HAS_WRONG_INDENTATION", "Has wrong indentation.", filename)
            is_valid = False

    if check_trailing_chars:
        if validate_trailing_chars(filename):
            log_error("HAS_TRAILING_CHARS", "Has trailing chars.", filename)
            is_valid = False

    # TODO: Continue implementation

    return is_valid

def is_valid_binary_kernel(filename, check_line_length=True, check_indentation=True, check_trailing_chars=True):
    extension = str(os.path.splitext(filename)[1]).lower()

    # Write comments to a fie
    comment_filename = filename + ".commnt"
    with open(comment_filename, 'w') as commnt_file:
        commnt_file.write(str(commnt_read(filename)))

    is_valid = is_valid_comment_file(comment_filename,
                                     check_line_length, check_indentation, check_trailing_chars)
    os.remove(comment_filename)

    if not has_valid_version_and_date_section(filename):
        is_valid = False

    if not has_valid_contact_section(filename):
        is_valid = False

    if is_spk_file(filename):
        if not is_valid_spk_kernel(filename):
            log_error("INVALID_SPK_KERNEL", "Invalid SPK kernel.", filename)
            is_valid = False

    elif is_ck_file(filename):
        if not is_valid_ck_kernel(filename):
            log_error("INVALID_CK_KERNEL", "Invalid CK kernel.", filename)
            is_valid = False

    # TODO: Do specific checks for Binary PCKs

    return is_valid

def is_valid_metakernel(mk_path):

    is_valid_mk = True
    prev_cwd = os.getcwd()

    abspath = os.path.abspath(mk_path)
    dname = os.path.dirname(abspath)
    mk_filename = os.path.basename(abspath)

    try:
        # Load the meta kernel then use KTOTAL to interrogate the SPICE
        # kernel subsystem.
        filename, extension = os.path.splitext(mk_filename)
        os.chdir(dname)
        spiceypy.furnsh(mk_filename)

        count = spiceypy.ktotal('ALL')

        # Loop over the number of files; interrogate the SPICE system
        # with spiceypy.kdata for the kernel names and the type.
        # 'found' returns a boolean indicating whether any kernel files
        # of the specified type were loaded by the kernel subsystem.
        # This example ignores checking 'found' as kernels are known
        # to be loaded.
        loaded_files = []
        for i in range(0, count):
            [k_file, ktype, source, handle] = spiceypy.kdata(i, 'ALL')
            k_filename = os.path.basename(k_file)
            if k_filename not in loaded_files:
                loaded_files.append(k_filename)
            else:
                log_error("WRONG_MK", "Duplicated kernel: " + str(k_file) + " in MK", mk_filename)
                is_valid_mk = False

        # Lets write the first 10000 variables and their values
        cvals = spiceypy.gnpool("*", 0, 10000)
        for cval in cvals:

            #
            # Use spiceypy.dtpool to return the dimension and type,
            # C (character) or N (numeric), of each pool
            # variable name in the cvals array. We know the
            # kernel data exists.
            #
            [dim, type] = spiceypy.dtpool(cval)

            #
            # Test character equality, 'N' or 'C'.
            #
            if type == 'N':

                #
                # If 'type' equals 'N', we found a numeric array.
                # In this case any numeric array will be an array
                # of double precision numbers ('doubles').
                # spiceypy.gdpool retrieves doubles from the
                # kernel pool.
                #
                dvars = spiceypy.gdpool(cval, 0, 10)

            elif type == 'C':

                #
                # If 'type' equals 'C', we found a string array.
                # spiceypy.gcpool retrieves string values from the
                # kernel pool.
                #
                cvars = spiceypy.gcpool(cval, 0, 10)

                if cval == "SKD_VERSION":
                    # Check filename aligned with SKD_VERSION for MKs with version
                    if is_versioned_mk(mk_filename):
                        if not filename.endswith(str(cvars[0])):
                            log_error("WRONG_MK", "Variable SKD_VERSION: " + str(cvars[0]) +
                                      " doesn't match MK filename: " + mk_filename, mk_filename)
                            is_valid_mk = False
                    else:
                        # Check that an MK with name MK_IDENTIFIER exists in the MKs path
                        related_mk = filename + "_" + str(cvars[0]) + extension
                        if not os.path.exists(related_mk):
                            log_error("WRONG_MK", "No MK ( " + related_mk + " ) with version found for "
                                      "SKD_VERSION: " + str(cvars[0]) + " at MK filename: " + mk_filename, mk_filename)
                            is_valid_mk = False

                elif cval == "MK_IDENTIFIER":
                    if is_versioned_mk(mk_filename):
                        # Check filename aligned with MK_IDENTIFIER for MKs with version
                        if str(cvars[0]) != filename:
                            log_error("WRONG_MK", "Variable MK_IDENTIFIER: " + str(cvars[0]) +
                                        " doesn't match MK filename: " + mk_filename, mk_filename)
                            is_valid_mk = False
                    else:
                        # Check that an MK with name MK_IDENTIFIER exists in the MKs path
                        related_mk = str(cvars[0]) + extension
                        if not os.path.exists(related_mk):
                            log_error("WRONG_MK", "No MK ( " + related_mk + " ) with version found for "
                                        "MK_IDENTIFIER: " + str(cvars[0]) + " at MK filename: " + mk_filename,
                                      mk_filename)
                            is_valid_mk = False

            else:

                #
                # This block should never execute.
                #
                log_error("WRONG_MK", "Unknown var type: " + str(type) + " at variable " + str(cval), mk_filename)
                is_valid_mk = False

        # Now unload the meta kernel. This action unloads all
        # files listed in the meta kernel.
        spiceypy.unload(mk_filename)

    except SpiceyError as err:
        log_error("WRONG_MK", "Raised exception: " + str(err) + " in mk: " + mk_path, mk_filename)
        is_valid_mk = False

    # Done. Unload the kernels.
    spiceypy.kclear()
    os.chdir(prev_cwd)

    return is_valid_mk

def has_valid_text_kernel_header(kernel_file):
    f = open(kernel_file).readlines()
    extension = str(os.path.splitext(kernel_file)[1]).lower()
    if not KERNEL_TEXT_HEADERS[extension] in f[0]:
        log_error("WRONG_KERNEL_HEADER", "WRONG KERNEL HEADER: " + f[0] + " instead of " +
                    KERNEL_TEXT_HEADERS[extension] + " for text kernel: " + kernel_file, kernel_file)
        return False

    return True

def has_valid_version_and_date_section(kernel_file):

    extension = str(os.path.splitext(kernel_file)[1]).lower()
    mandatory = extension in HAS_VERSION_AND_DATE_SECTION_EXTENSIONS

    if extension == ".tsc":
        if "_v" not in os.path.basename(kernel_file).lower():
            # Ignore this check for SCLK Kernels with no version in the filename, such as: integral_fict_20220208.tsc
            return True

    text = get_section_text_from_kernel_comments(kernel_file, "Version and Date")
    if not len(text):
        if mandatory:
            log_error("WRONG_VERSION_SECTION", "No 'Version and Date' section found in kernel: " + kernel_file,
                      kernel_file)
            return False
        else:
            return True

    multiplier = 1
    try:
        kernel_version = int(get_kernel_version(kernel_file))
        if "_v" not in os.path.basename(kernel_file).lower():
            # In case that filenames has the form: rssd0003.tf or similar, we need to multiply
            # the version by 10 because in 'Version and Date' section shall appear like 3.0 -> 30
            multiplier = 10
    except Exception as ex:
        log_error("WRONG_VERSION_SECTION", "Could not obtain kernel version from kernel: " + kernel_file
                  + " , ex: " + str(ex), kernel_file)
        return False

    tmp_kernel_version = kernel_version * multiplier
    # Check that all defined versions in section are properly declared
    for line in text.splitlines():
        if line.strip().startswith("Version"):

            if "-draft" in line:
                # Ignore draft versions
                continue

            try:
                version = int(line.split("--")[0].strip().replace("Version", "").strip().replace(".", ""))

                if version > tmp_kernel_version:
                    log_error("WRONG_VERSION_SECTION",
                              "Version in 'Version and Date' section is greater than expected "
                                "in kernel: " + kernel_file + " with version: " + str(kernel_version)
                                + ", at line: " + line, kernel_file)
                    return False

                if version < tmp_kernel_version:
                    log_error("WRONG_VERSION_SECTION",
                              "Version in 'Version and Date' section is smaller than expected "
                                "in kernel: " + kernel_file + " with version: " + str(kernel_version)
                                + ", at line: " + line, kernel_file)
                    return False

                tmp_kernel_version = tmp_kernel_version - multiplier

            except Exception:
                log_error("WRONG_VERSION_SECTION",
                          "Wrong version text format in 'Version and Date' section found in kernel: "
                          + kernel_file + " with version: " + str(kernel_version) + ", at line: " + line, kernel_file)
                return False

    if tmp_kernel_version > -1:
        log_error("WRONG_VERSION_SECTION",
                  "Missing versions in 'Version and Date' section in kernel: " + kernel_file, kernel_file)
        return False

    return True

def has_valid_contact_section(kernel_file, mandatory=True):

    text = get_section_text_from_kernel_comments(kernel_file, "Contact Information")
    if not len(text):
        if mandatory:
            log_error("WRONG_CONTACT", "No 'Contact Information' section found", kernel_file)
            return False
        else:
            return True

    if CONTACT not in text:
        log_error("WRONG_CONTACT", "'" + CONTACT + "' not found in 'Contact Information' section", kernel_file)
        return False

    return True

def get_skd_version(skd_path):

    version_file = os.path.join(skd_path, "version")
    if not os.path.exists(version_file):
        raise Exception("Version file not found at path: " + version_file)

    version = read_all_text(version_file).strip()
    if len(version) != 4 or not version.startswith("v"):
        raise Exception("Wrong formatted version: '" + version + "' found at: " + version_file)

    return version

def has_extension(path, extension):
    fk_filename = os.path.basename(path)
    return str(os.path.splitext(fk_filename)[1]).lower() == extension

def is_fk_file(path):
    return has_extension(path, ".tf")

def is_ik_file(path):
    return has_extension(path, ".ti")

def is_mk_file(path):
    return has_extension(path, ".tm")

def is_spk_file(path):
    return has_extension(path, ".bsp")

def is_ck_file(path):
    return has_extension(path, ".bc")

def is_versioned_mk(mk_path):
    mk_filename = os.path.basename(mk_path)
    if not is_mk_file(mk_path):
        raise Exception("File is not a meta-kernel file: " + mk_path)

    return "_v" in str(os.path.splitext(mk_filename)[0]).lower() and not is_local_mk(mk_path)

def is_local_mk(mk_path):
    if not is_mk_file(mk_path):
        raise Exception("File is not a meta-kernel file: " + mk_path)

    return str(os.path.splitext(mk_path)[0]).lower().endswith("local")

def get_versions_history_from_release_notes_file(rel_notes_file):

    text = get_section_text_from_kernel_comments(rel_notes_file, "Appendix: Release History")
    if not len(text):

        text = get_section_text_from_kernel_comments(rel_notes_file, "Release History")
        if not len(text):
            log_error("WRONG_RELEASE_NOTES",
                      "No 'Release History' section found in kernel: " + rel_notes_file, rel_notes_file)
            return None

    versions = []
    # Check that all defined versions in section are properly declared
    indentation = -1
    for line in text.splitlines():
        if indentation < 0:
            if len(line.strip()) > 0:
                indentation = len(line) - len(line.lstrip())

        if indentation >= 0:
            if len(line) - len(line.lstrip()) == indentation:
                tokens = line.strip().split(" ")
                if len(tokens) == 4:
                    if tokens[3].startswith("v") and len(tokens[3].split(".")) == 3:
                        versions.append(tokens[3])

    return versions

def check_release_notes_version(rel_notes_file):

    basename = os.path.basename(rel_notes_file)
    filename, extension = os.path.splitext(basename)

    text = read_all_text(rel_notes_file)
    first_line = text.splitlines()[0]
    version = first_line.split()[-1]

    is_valid = True
    if not filename.endswith("skd_current"):
        if not filename.endswith("_skd_" + version.replace("v", "").replace(".", "")):
            log_error("WRONG_RELEASE_NOTES",
                      "Filename versions: " + rel_notes_file + " doesn't match the version: " + version
                        + " at line: " + first_line, rel_notes_file)
            is_valid = False

    # First paragraph shall contain "(version)"
    ver_text = "(" + version.replace("v", "") + ")"
    if ver_text not in text:
        log_error("WRONG_RELEASE_NOTES", "Text : " + ver_text + " not found in first paragraph.", rel_notes_file)
        is_valid = False

    notes = get_section_text_from_kernel_comments(rel_notes_file, "Notes")
    if not len(notes):
        log_error("WRONG_RELEASE_NOTES", "No 'Notes' section found in kernel.", rel_notes_file)

    elif version not in notes:
        log_error("WRONG_RELEASE_NOTES", "Version : " + version + " not found in Notes section.", rel_notes_file)
        is_valid = False

    return is_valid

def is_valid_frames_kernel(fk_path):

    try:
        data_text, comments = get_text_and_data_from_kernel(fk_path)
    except Exception as ex:
        log_error("DATA_AND_COMMENTS",
                  "Obtaining data text from: " + fk_path + " , exception: " + str(ex), fk_path)
        return False

    if not len(data_text):
        log_error("DATA_AND_COMMENTS",
                  "No data sections found in kernel: " + fk_path, fk_path)
        return False

    if not len(comments):
        log_error("DATA_AND_COMMENTS",
                  "No text comments found in kernel: " + fk_path, fk_path)
        return False

    # Don't continue if is a PINPOINT FK
    if "This file was created by PINPOINT." in comments:
        log_info("DATA_AND_COMMENTS",
                  "Not all FK checks were performed because is a PINPOINT FK: " + fk_path, fk_path)
        return True

    sections_map = get_sections_map_from_kernel_comments(comments)
    if sections_map is None:
        log_error("DATA_AND_COMMENTS",
                  "Could not obtain sections from kernel: " + fk_path, fk_path)
        return False

    # Check required sections
    all_required_section_found = check_required_sections(sections_map, REQUIRED_SECTIONS["FK"], fk_path)

    # Check NAIF IDs if any is found
    valid_ids = check_naif_id_associations(data_text, sections_map, fk_path)

    # Check frames definitions
    valid_frames = check_frame_definitions(data_text, sections_map, fk_path)

    return all_required_section_found and valid_ids and valid_frames

def is_valid_instruments_kernel(ik_path):

    try:
        data_text, comments = get_text_and_data_from_kernel(ik_path)
    except Exception as ex:
        log_error("DATA_AND_COMMENTS",
                  "Obtaining data text from: " + ik_path + " , exception: " + str(ex), ik_path)
        return False

    if not len(data_text):
        log_error("DATA_AND_COMMENTS",
                  "No data sections found in kernel: " + ik_path, ik_path)
        return False

    if not len(comments):
        log_error("DATA_AND_COMMENTS",
                  "No text comments found in kernel: " + ik_path, ik_path)
        return False

    sections_map = get_sections_map_from_kernel_comments(comments)
    if sections_map is None:
        log_error("DATA_AND_COMMENTS",
                  "Could not obtain sections from kernel: " + ik_path, ik_path)
        return False

    # Check required sections
    all_required_section_found = check_required_sections(sections_map, REQUIRED_SECTIONS["IK"], ik_path)

    # Check NAIF IDs if any is found
    valid_ids = check_naif_id_associations(data_text, sections_map, ik_path)

    # Check instruments definitions
    valid_instruments = check_instruments_definitions(data_text, sections_map, ik_path)

    return all_required_section_found and valid_ids and valid_instruments

def is_valid_spk_kernel(spk_path):

    try:
        data_text, comments = get_text_and_data_from_kernel(spk_path)
    except Exception as ex:
        log_error("DATA_AND_COMMENTS",
                  "Obtaining data text from: " + spk_path + " , exception: " + str(ex), spk_path)
        return False

    if not len(comments):
        log_error("DATA_AND_COMMENTS",
                  "No text comments found in kernel: " + spk_path, spk_path)
        return False

    if len(data_text):

        sections_map = get_sections_map_from_kernel_comments(comments)
        if sections_map is None:
            log_error("DATA_AND_COMMENTS",
                      "Could not obtain sections from kernel: " + spk_path, spk_path)
            return False

        if get_section_from_sections_map("@IN@PINPOINT", sections_map)[0] is not None:
            # Is a PINPOINT Kernel file

            # Check required sections
            all_required_section_found = check_required_sections(sections_map,
                                                                 REQUIRED_SECTIONS["SPK_PINPOINT"],
                                                                 spk_path)

            # Check sites definitions
            valid_sites = check_sites_definitions(data_text, sections_map, spk_path)

            return all_required_section_found and valid_sites

        elif get_section_from_sections_map("@IN@OEM2SPK", sections_map)[0] is not None:
            # Is a OEM2SPK Kernel file

            # Check required sections
            all_required_section_found = check_required_sections(sections_map,
                                                                 REQUIRED_SECTIONS["SPK_OEM2SPK"],
                                                                 spk_path)
            return all_required_section_found

        elif get_section_from_sections_map("@IN@MKSPK", sections_map)[0] is not None:
            # Is a MKSPK Kernel file

            # Check required sections
            all_required_section_found = check_required_sections(sections_map,
                                                                 REQUIRED_SECTIONS["SPK_MKSPK"],
                                                                 spk_path)
            return all_required_section_found

        else:
            log_error("DATA_AND_COMMENTS",
                      "SPK type not supported: " + spk_path, spk_path)
            return False

    else:
        # The SPK doesn't have /begindata sections, such as some NAIF SPKs and similar.
        return True

def is_valid_ck_kernel(ck_path):

    try:
        data_text, comments = get_text_and_data_from_kernel(ck_path)
    except Exception as ex:
        log_error("DATA_AND_COMMENTS",
                  "Obtaining data text from: " + ck_path + " , exception: " + str(ex), ck_path)
        return False

    if not len(comments):
        log_error("DATA_AND_COMMENTS",
                  "No text comments found in kernel: " + ck_path, ck_path)
        return False

    sections_map = get_sections_map_from_kernel_comments(comments)
    if sections_map is None:
        log_error("DATA_AND_COMMENTS",
                  "Could not obtain sections from kernel: " + ck_path, ck_path)
        return False

    if get_section_from_sections_map("@IN@orientation", sections_map)[0] is not None:
        # Is a Fixed attitude CK
        all_required_section_found = check_required_sections(sections_map,
                                                             REQUIRED_SECTIONS["CK_FIXED"],
                                                             ck_path)
        return all_required_section_found

    elif get_section_from_sections_map("Directions", sections_map)[0] is not None\
            or get_section_from_sections_map("Orientations", sections_map)[0] is not None\
            or get_section_from_sections_map("Schedule", sections_map)[0] is not None:
        # Is a predickt CK
        all_required_section_found = check_required_sections(sections_map,
                                                             REQUIRED_SECTIONS["CK_PREDICKT"],
                                                             ck_path)
        return all_required_section_found

    else:
        # Check required sections
        all_required_section_found = check_required_sections(sections_map,
                                                             REQUIRED_SECTIONS["CK"],
                                                             ck_path)
        return all_required_section_found

def check_required_sections(sections_map, required_sections, file_path):
    all_required_section_found = True

    for section_name in required_sections:
        sec_name, sec_text = get_section_from_sections_map(section_name, sections_map)

        if sec_name is None:
            log_error("MISSING_SECTION",
                      "Required section not found: '" + section_name + "' at " + file_path, file_path)
            all_required_section_found = False

        if not len(str(sec_text).strip()):
            log_error("MISSING_SECTION",
                      "Required section is empty: '" + section_name + "' at " + file_path, file_path)
            all_required_section_found = False

    return all_required_section_found

def check_naif_id_associations(data_text, sections_map, kernel_path):
    # Check NAIF IDs if any is found
    valid_ids = True

    try:
        naif_ids = get_naif_ids_from_text(data_text)
        FOUND_NAIF_IDS.extend(naif_ids)
    except Exception as ex:
        log_error("NAIF_IDS",
                  "Error while obtaining NAIF IDs from: " + kernel_path + " , exception: " + str(ex), kernel_path)
        return False

    if len(naif_ids):

        naif_id_section_patterns = ["@EW@NAIF ID Codes",
                                    "@EW@NAIF ID Codes to Name Mapping",
                                    "@EW@NAIF ID Codes -- Definitions"]
        naif_id_sections = get_matched_sections(sections_map, naif_id_section_patterns)

        # TODO: Remove this check when next TODO is implemented, for the moment just look for name and syno
        for naif_id in naif_ids.keys():
            for naif_id_section in naif_id_sections:
                # Note that synonyms array also contains the body_name
                if not (check_naif_id_in_section_text(naif_id, naif_ids[naif_id]["synonyms"],
                                                      naif_id_section[0], naif_id_section[1], kernel_path)):
                    valid_ids = False

    """
    # TODO: Continue implementation of commented code, it parses the NAIF Ids from the description tables
    #       and checks that are consistent with naif id code associations.
    #       For the moment is not working because the are still work in progress for function:
    #           files.get_naif_ids_from_text_with_tables()

    naif_ids_table1 = get_naif_ids_from_text_with_tables(naif_id_codes_sec_text)
    naif_ids_table2 = get_naif_ids_from_text_with_tables(naif_id_codes_map_sec_text)

    naif_ids_set = set(naif_ids.keys())
    naif_ids_table1_set = set(naif_ids_table1.keys())
    naif_ids_table2_set = set(naif_ids_table2.keys())

    if naif_ids_set != naif_ids_table1_set or naif_ids_set != naif_ids_table2_set:

        all_naif_ids = naif_ids_set.union(naif_ids_table1_set).union(naif_ids_table2_set)

        for naif_id in all_naif_ids:

            valid_id = True

            if naif_id not in naif_ids:
                print("ERROR: NAIF ID Code: " + str(naif_id) + " not found in NAIF_BODY_CODEs at " + fk_path)
                valid_id = False

            if naif_id not in naif_ids_table1:
                print("ERROR: NAIF ID Code: " + str(naif_id) + " not found in section " +
                        "'NAIF ID Codes' section at " + fk_path)
                valid_id = False

            if naif_id not in naif_ids_table2:
                print("ERROR: NAIF ID Code: " + str(naif_id) + " not found in section " +
                        "'NAIF ID Codes to Name Mapping' section at " + fk_path)
                valid_id = False

            if valid_id:

                set1 = set(naif_ids[naif_id]["synonyms"])
                set2 = set(naif_ids_table1[naif_id]["synonyms"])
                set3 = set(naif_ids_table2[naif_id]["synonyms"])

                all_synonyms = set1.union(set2).union(set3)
                for synonym in all_synonyms:

                    if synonym not in naif_ids[naif_id]["synonyms"]:
                        print("ERROR: NAIF BODY Name: " + str(synonym) + " not found in NAIF_BODY_CODEs "
                                                                         "for NAIF Id: " + str(
                            naif_id) + " at " + fk_path)
                        valid_id = False

                    if synonym not in naif_ids_table1[naif_id]["synonyms"]:
                        print("ERROR: NAIF BODY Name: " + str(synonym) + " not found in 'NAIF ID Codes' section "
                                                                         "for NAIF Id: " + str(
                            naif_id) + " at " + fk_path)
                        valid_id = False

                    if synonym not in naif_ids_table2[naif_id]["synonyms"]:
                        print("ERROR: NAIF BODY Name: " + str(
                            synonym) + " not found in 'NAIF ID Codes to Name Mapping' section "
                                       "for NAIF Id: " + str(naif_id) + " at " + fk_path)
                        valid_id = False

            valid_ids = valid_ids and valid_id
    """

    return valid_ids

def check_naif_id_in_section_text(naif_id, synonyms, section_name, section_text, kernel_path):

    naif_id_is_valid = True

    if str(naif_id) not in section_text:

        # Check if we shall look for naif_id ended in NN like: -12151NN*
        if len(synonyms) == 1:
            last_name_part = synonyms[0].split("_")[-1]
            if is_number(last_name_part):
                n_digits = len(last_name_part)
                generic_naif_id = str(naif_id)[0:-n_digits] + "N" * len(last_name_part)
                generic_synonym = synonyms[0][0:-n_digits] + "N" * len(last_name_part)
                return check_naif_id_in_section_text(generic_naif_id, [generic_synonym],
                                                     section_name, section_text, kernel_path)

        log_error("NAIF_IDS",
                  "NAIF ID Code: " + str(naif_id) + " not found at section: '" + section_name, kernel_path)
        naif_id_is_valid = False

    for synomyn in synonyms:
        if synomyn not in section_text:
            # We need to check that the synonym has not been chunked in two lines because it has a long name
            if synomyn not in " ".join(section_text.replace("\n", " ").split()):
                log_error("NAIF_IDS",
                          "NAIF BODY Name: " + str(synomyn) + " not found at section: '" + section_name, kernel_path)
                naif_id_is_valid = False

    return naif_id_is_valid

def check_frame_definitions(data_text, sections_map, fk_path):
    try:
        frames = get_frames_definitions_from_text(data_text)

        for frame_id in frames:
            FOUND_FRAME_IDS.append(frame_id)
            if "frame_name" in frames[frame_id]:
                FOUND_FRAME_NAMES.append(frames[frame_id]["frame_name"])

    except Exception as ex:
        log_error("WRONG_DEFINITIONS",
                  "Error while obtaining frame definitions from: " + fk_path + " , exception: " + str(ex), fk_path)
        return False

    not_found_ids = []
    frames_are_valid = True
    if len(frames):

        frames_section_patterns = ["@EW@Mission Frames"]
        # TODO: Include specific frame definition sections to frames_section_patterns,
        #       eg: "JUICE Medium Gain Antenna Frames" ...

        frame_sections = get_matched_sections(sections_map, frames_section_patterns)

        # Check that frame Ids and Frame names are in the comments sections
        for frame_id in frames.keys():
            for frame_section in frame_sections:

                if str(frame_id) not in frame_section[1]:
                    log_error("MISALIGNED_SECTION",
                              "FRAME ID Code: " + str(frame_id) + " not found at section: '" + frame_section[0]
                              + "' at " + fk_path, fk_path)
                    frames_are_valid = False
                    not_found_ids.append(frame_id)

                if "frame_name" in frames[frame_id]:
                    frame_name = frames[frame_id]["frame_name"]
                    if frame_name not in frame_section[1]:
                        log_error("MISALIGNED_SECTION",
                                  "FRAME NAME: " + frame_name + " not found at section: '" + frame_section[0]
                                  + "' at " + fk_path, fk_path)
                        frames_are_valid = False

        # Validate each frame definition
        keyword_indent = None  # Must be the same for all frames

        for frame_id in frames:

            if frame_id in not_found_ids:
                # Skip because we already know that its definition is wrong:
                continue

            keyword_indent = check_keywords_indentation(frames[frame_id]["keywords"], keyword_indent, fk_path)

            frames_is_valid = check_kernel_keywords(frames[frame_id], "frame_class_Any",
                                                    FRAME_DEFINITION_KEYWORDS, fk_path)
            if not frames_is_valid:
                frames_are_valid = False

    return frames_are_valid

def check_keywords_indentation(keywords, keyword_indent, path):

    equal_indent = None  # Must be the same inside each frame
    value_indent = None  # Must be the same inside each frame

    for keyword_key in keywords:
        keyword_data = keywords[keyword_key]

        # Check indentation
        if equal_indent is None \
                or ("indent_break" in keyword_data and keyword_data["indent_break"]):
            equal_indent = keyword_data["equal_indent"]

            value_indent = keyword_data["value_indent"]
            if is_number(keyword_data["value"]) and float(keyword_data["value"]) >= 0.0:
                # In case of first value is a positive number, is shall have an extra blank
                # so our indentation is one char minus
                value_indent = value_indent - 1

            if keyword_indent is None:
                keyword_indent = keyword_data["keyword_indent"]
        else:

            tmp_value_indent = keyword_data["value_indent"]
            if is_number(keyword_data["value"]) and float(keyword_data["value"]) >= 0.0:
                # In case of value where is a positive number, is shall have an extra blank
                # so our indentation is one char minus
                tmp_value_indent = tmp_value_indent - 1

            if keyword_data["keyword_indent"] != keyword_indent:
                log_warn("WRONG_INDENTATION",
                          "WRONG INDENTATION OF KEYWORD: '" + keyword_data["var"] + "' at line: '" +
                          keyword_data["line"] + "' at " + path, path)
            elif keyword_data["equal_indent"] != equal_indent:
                log_warn("WRONG_INDENTATION",
                         "WRONG INDENTATION OF '=' at line: '" + keyword_data["line"] + "' at " + path, path)
            elif tmp_value_indent != value_indent:
                log_warn("WRONG_INDENTATION",
                         "WRONG INDENTATION OF '" + keyword_data["value"] + "' at line: '" +
                         keyword_data["line"] + "' at " + path, path)

    return keyword_indent

def check_kernel_keywords(def_obj, keywords_ref_key, keywords_ref_map, path):
    keywords_valid = True

    keywords_ref = keywords_ref_map[keywords_ref_key]

    keywords = def_obj["keywords"]
    for keyword_ref_data in keywords_ref:

        keyword_ref = keyword_ref_data["keyword"]
        if keyword_ref not in keywords:

            if "optional" in keyword_ref_data:
                if eval(keyword_ref_data["optional"]):
                    continue  # Ignore this keyword in this case

            log_error("WRONG_KEYWORD",
                      "MISSING KEYWORD AT DEFINITION: " + keyword_ref + " not found at: \n" + def_obj["definition"],
                      path)
            keywords_valid = False
            continue

        keyword_data = keywords[keyword_ref]

        # Check keyword validity
        var_ref = replace_tokens(keyword_ref_data["keyword"], keyword_data, def_obj)
        if keyword_data["var"] != var_ref:
            log_error("WRONG_KEYWORD",
                      "WRONG KEYWORD AT DEFINITION: " + keyword_data["var"] + " expected: '" + var_ref, path)
            keywords_valid = False

        # Check line order
        if "line_nr" in keyword_ref_data:
            if keyword_data["line_nr"] != keyword_ref_data["line_nr"]:
                log_warn("WRONG_KEYWORD_ORDER",
                          "WRONG LINE ORDER FOR KEYWORD '" + keyword_data["var"] + "' " +
                          "found at line nr: " + str(keyword_data["line_nr"]) +
                          " expected at line nr: " + str(keyword_ref_data["line_nr"]), path)

        # Check proper value
        value = ""
        if "value" in keyword_ref_data:

            value_ref = keyword_ref_data["value"]
            value = keyword_data["value"]  # To have value variable inside eval context
            if isinstance(value, str):
                # Check number of "'" in string
                if "'" in value and len(value.split("'")) % 2 != 1:
                    log_error("WRONG_KEYWORD", "WRONG NUMBER OF \"'\" FOUND AT LINE: '" + keyword_data["line"], path)
                    keywords_valid = False

                # Remove "'" from sting
                value = value.replace("'", "")

            if "{" in value_ref and not value_ref.startswith("="):
                value_ref = replace_tokens(value_ref, keyword_data, def_obj)
                if value != value_ref:
                    log_error("WRONG_KEYWORD",
                              "WRONG VALUE FOUND AT LINE: '" + keyword_data["line"] + "' " +
                              "found: '" + str(value) + "' expected: '" + str(value_ref), path)
                    keywords_valid = False

            elif value_ref.startswith("["):
                value_ref = eval(value_ref)
                if value not in value_ref:
                    log_error("WRONG_KEYWORD",
                              "WRONG VALUE FOUND AT LINE: '" + keyword_data["line"] + "' " +
                              "found: '" + str(value) + "' expected: '" + str(value_ref), path)
                    keywords_valid = False

            elif value_ref.startswith("="):

                result = eval(replace_tokens(value_ref[1:], keyword_data, def_obj))

                reason = None
                if isinstance(result, tuple):
                    valid_value = result[0]
                    reason = result[1]
                else:
                    valid_value = result

                if not valid_value:

                    reason_text = ""
                    if reason is not None:
                        reason_text = ", reason: " + reason + " "

                    log_error("WRONG_KEYWORD",
                              "WRONG VALUE FOUND AT LINE: '" + keyword_data["line"] + "' " + reason_text +
                              "found: '" + str(value) + "' expected: '" + str(value_ref), path)
                    keywords_valid = False

            elif value_ref == "NAIF_ID":

                if not is_naif_id(value):
                    log_warn("WRONG_KEYWORD",
                              "WARNING: WRONG VALUE FOUND AT LINE: '" + keyword_data["line"] + "' " +
                              "found: '" + str(value) + "' expected any of defined NAIF IDs " +
                              "at the validated kernels or any of the SPICE BUILT-IN BODY IDs.\n" +
                              "Check if this BODY ID has been defined in other kernel.\n" +
                              "Warning raised at " + path, path)

            elif value_ref == "FRAME_NAME" \
                    or value_ref == "FRAME_NAME_LIST":

                frame_names = []
                if value_ref == "FRAME_NAME_LIST":
                    if not is_spice_vector(value, str, check_duplicates=True):
                        log_error("WRONG_KEYWORD",
                                  "WRONG FRAME NAME LIST FOUND AT LINE: '" + keyword_data["line"] + "' " +
                                  "found: '" + str(value), path)
                        keywords_valid = False
                    else:
                        frame_names.extend(spice_vector_to_tuple(value, str))
                else:
                    frame_names.append(value)

                for frm_name in frame_names:
                    if not is_frame_name(frm_name):
                        log_warn("WRONG_KEYWORD",
                                  "WARNING: WRONG FRAME NAME FOUND AT LINE: '" + keyword_data["line"] + "' " +
                                  "found: '" + str(frm_name) + "' expected any of defined FRAME NAMES " +
                                  "at the validated FKs or any of the SPICE BUILT-IN FRAMES.\n" +
                                  "Check if this FRAME NAME has been defined in other kernel.\n" +
                                  "Warning raised at " + path, path)

            else:
                raise NotImplementedError("Reference value not supported: " + value_ref)

        # Iterate over sub keyword in case of defined
        if "sub_keywords_key" in keyword_ref_data:

            sub_keywords_key = eval(keyword_ref_data["sub_keywords_key"][1:])
            if sub_keywords_key in keywords_ref_map:
                sub_keywords_valid = check_kernel_keywords(def_obj, sub_keywords_key, keywords_ref_map, path)
                if not sub_keywords_valid:
                    keywords_valid = False

            else:
                log_warn("WRONG_KEYWORD",
                         "WARNING: Could not check sub_keywords for key: " + sub_keywords_key
                         + ", Warning raised at " + path, path)

    return keywords_valid

def replace_tokens(text, keyword_data, spice_obj):

    for token in REPLACE_TOKENS:
        token_s = "{" + token + "}"
        if token_s in text:
            token_v = str(spice_obj[token]) \
                if token in spice_obj \
                else str(keyword_data[token])
            text = text.replace(token_s, token_v)
            if "{" not in text:
                break

    return text

def is_naif_id(naif_id):
    try:
        if not isinstance(naif_id, int):
            naif_id = int(naif_id)
    except:
        return False

    if naif_id not in FOUND_NAIF_IDS:
        try:
            # TODO: Load MK to have all defined NAIF IDs in the pool
            name = spiceypy.bodc2n(naif_id)
        except NotFoundError:
            return False

    return True

def is_frame_id(frame_id):
    try:
        if not isinstance(frame_id, int):
            frame_id = int(frame_id)
    except:
        return False

    if frame_id not in FOUND_FRAME_IDS:
        try:
            # TODO: Load MK to have all defined FRAME IDS in the pool
            frame_name = spiceypy.frmnam(frame_id)
        except NotFoundError:
            return False

    return True

def is_frame_name(frame_name):
    if not isinstance(frame_name, str):
        return False

    if frame_name not in FOUND_FRAME_NAMES:
        try:
            # TODO: Load MK to have all defined FRAME NAMES in the pool
            frame_id = spiceypy.namfrm(frame_name)
        except NotFoundError:
            return False

    return True

def spice_vector_to_tuple(text, elem_type):
    if "(" not in text or ")" not in text:
        raise Exception("Missing parenthesis")

    if "," not in text:
        # Eg: ( 1.0 4.0 5.0 )
        elems = text.replace("(", "").replace(")", "").split()
        new_elems = ["'" + elem + "'"
                     if not is_number(elem) and "'" not in elem
                     else elem
                     for elem in elems]

        text = "(" + ",".join(new_elems) + ")"

    elif "\n" in text:
        # Add "," to the line endings if not present. Eg: 0.00000,  1.00000,  0.00000
        new_text = ""
        comma_endings = None
        lines = text.splitlines()
        num_lines = len(lines)
        for line_idx in range(num_lines):
            line = lines[line_idx]
            if line_idx < num_lines - 1 \
                    and line.strip() != "(":
                next_line = lines[line_idx + 1]
                tmp_comma_endings = line.rstrip().endswith(",")
                if comma_endings is None:
                    comma_endings = tmp_comma_endings
                elif comma_endings != tmp_comma_endings and next_line.strip() != ")":
                    raise Exception("Different line endings with or without ','")

                new_text += line + ("," if not tmp_comma_endings
                                           and not line.rstrip().endswith(")")
                                           and next_line.strip() != ")"
                                    else "")
            else:
                new_text += line

        text = new_text

    if elem_type == "date_str":
        dates_texts = text.replace("(", "").replace(")", "").split(",")
        dates = []
        for date in dates_texts:
            dates.append(date.strip().replace("@", "'@") + "'")
        text = "(" + ",".join(dates) + ")"

    matrix = eval(text)
    if "," in text:
        if not isinstance(matrix, tuple):  # Eg: (0.0, 1.0, 2.0)
            raise Exception("Wrong (f0, ..) format")
    else:
        if not has_valid_type(matrix, elem_type):  # Eg: (0.0)
            raise Exception("Wrong (f0) format")
        else:
            matrix = [matrix]

    return matrix

def is_spice_vector(text, elem_type, size=None, values_range=None, check_duplicates=False):
    try:
        matrix = spice_vector_to_tuple(text, elem_type)

        if size is not None:
            if isinstance(size, int):
                if len(matrix) != size:
                    return False, "Wrong length or missing comma, expected " + str(size) + " elements."
            elif isinstance(size, str):
                if not eval(size):
                    return False, "Wrong size or missing comma."
            else:
                raise NotImplementedError("Unsupported size type at is_spice_vector, type: " + str(type(size)))

        for elem in matrix:
            if not has_valid_type(elem, elem_type):
                return False, "Wrong type for element: " + str(elem) + ", expected any of " + \
                              str(elem_type) + " types."

            if values_range is not None:
                if elem not in values_range:
                    return False, "Wrong value for element: " + str(elem) + ", expected any of " + \
                              str(values_range) + " range."

        if check_duplicates:
            if len(matrix) != len(set(matrix)):
                return False, "Duplicated values"

    except Exception as ex:
        return False, str(ex)

    return True

def has_valid_type(elem, elem_type):
    if elem_type == "date_str":
        elem_type = str

    elem_types = [elem_type]
    if isinstance(elem_type, list):
        elem_types = elem_type

    valid_type = False
    for e_type in elem_types:
        if isinstance(elem, e_type):
            valid_type = True
            break
    return valid_type

def is_number(value_s):
    try:
        val = float(value_s)
        return True
    except:
        return False

def check_instruments_definitions(data_text, sections_map, ik_path):
    try:
        instruments = get_instruments_definitions_from_text(data_text)
    except Exception as ex:
        log_error("WRONG_DEFINITIONS",
                  "Obtaining instruments definitions from: " + ik_path + " , exception: " + str(ex), ik_path)
        return False

    ins_are_valid = True
    if len(instruments):

        ins_section_patterns = ["@IN@Naming Conventions"]
        ins_sections = get_matched_sections(sections_map, ins_section_patterns)

        # Check that instrument Ids and name are in the comments sections
        for ins_id in instruments:
            for section in ins_sections:

                if str(ins_id) not in section[1]:
                    log_error("MISALIGNED_SECTION",
                              "INSTRUMENT ID Code: " + str(ins_id) + " not found at section: '" + section[0], ik_path)
                    ins_are_valid = False

                if "name" in instruments[ins_id]:
                    ins_name = instruments[ins_id]["name"]
                    if ins_name not in section[1]:
                        log_error("MISALIGNED_SECTION",
                                  "INSTRUMENT NAME: " + ins_name + " not found at section: '" + section[0], ik_path)
                        ins_are_valid = False

        # Validate each instrument definition
        keyword_indent = None  # Must be the same for all instruments

        for ins_id in instruments:

            keyword_indent = check_keywords_indentation(instruments[ins_id]["keywords"], keyword_indent, ik_path)

            ins_is_valid = check_kernel_keywords(instruments[ins_id], "instrument_Any",
                                                 INSTRUMENT_DEFINITION_KEYWORDS, ik_path)

            if not ins_is_valid:
                ins_are_valid = False

    return ins_are_valid

def check_sites_definitions(data_text, sections_map, spk_path):
    try:
        sites = get_sites_definitions_from_text(data_text)
    except Exception as ex:
        log_error("WRONG_DEFINITIONS",
                  "Obtaining sites definitions from: " + spk_path + " , exception: " + str(ex), spk_path)
        return False

    sites_are_valid = True
    if len(sites):

        sites_section_patterns = ["Structure location specification -- Definitions",
                                  "Coordinates"]
        sites_sections = get_matched_sections(sections_map, sites_section_patterns)

        # Check that site Ids and site names are in the comments sections
        for site_name in sites.keys():

            id_code = None
            if "{name}_IDCODE" in sites[site_name]["keywords"]:
                id_code = sites[site_name]["keywords"]["{name}_IDCODE"]["value"]
                if is_number(id_code) and id_code not in FOUND_NAIF_IDS:
                    FOUND_NAIF_IDS.append(int(id_code))

            for section in sites_sections:

                if section[0] != "Coordinates":
                    if site_name not in section[1]:
                        log_error("MISALIGNED_SECTION",
                                  "SITE Name: '" + site_name + "' not found at section: '" + section[0], spk_path)
                        sites_are_valid = False

                if id_code is not None:
                    if str(id_code) not in section[1]:
                        log_error("MISALIGNED_SECTION",
                                  "SITE IDCODE: " + str(id_code) + " not found at section: '" + section[0], spk_path)
                        sites_are_valid = False

                if "{name}_FRAME" in sites[site_name]["keywords"]:
                    frmae_name = sites[site_name]["keywords"]["{name}_FRAME"]["value"].replace("'", "")
                    if frmae_name not in section[1]:
                        log_error("MISALIGNED_SECTION",
                                  "SITE FRAME: '" + frmae_name + "' not found at section: '" + section[0], spk_path)
                        sites_are_valid = False

                if section[0] == "Coordinates":
                    if "{name}_CENTER" in sites[site_name]["keywords"]:
                        center = sites[site_name]["keywords"]["{name}_CENTER"]["value"]
                        if str(center) not in section[1]:
                            log_error("MISALIGNED_SECTION",
                                      "SITE CENTER: " + str(center) + " not found at section: '" + section[0], spk_path)
                            sites_are_valid = False

        # Validate each site definition
        keyword_indent = None  # Must be the same for all instruments

        for site_name in sites:

            keyword_indent = check_keywords_indentation(sites[site_name]["keywords"], keyword_indent, spk_path)

            site_is_valid = check_kernel_keywords(sites[site_name], "sites_Any",
                                                  PINPOINT_DEFINITION_KEYWORDS, spk_path)

            if not site_is_valid:
                sites_are_valid = False

    return sites_are_valid

def get_matched_sections(sections_map, section_patterns):
    sections = []
    for sec_pattern in section_patterns:
        sec_name, sec_text = get_section_from_sections_map(sec_pattern, sections_map)
        if sec_name is not None:
            sections.append([sec_name, sec_text])
    return sections


import time
from collections import namedtuple
import BigWorld
from constants import IS_BASEAPP
from debug_utils import *
ENTITY_TYPE_ACCOUNT = 0
ENTITY_TYPE_CLAN = 1
ENTITY_TYPE_NAMES_BY_IDS = ('Account', 'Clan')
ENTITY_TYPE_IDS_BY_NAMES = {'Account': ENTITY_TYPE_ACCOUNT,
 'Clan': ENTITY_TYPE_CLAN}
ENTITY_TYPE_IDS = (ENTITY_TYPE_ACCOUNT, ENTITY_TYPE_CLAN)
OFFER_SELL = 0
_OFFER_KIND_MASK = 192
SRC_WARE_GOLD = 0
SRC_WARE_CREDITS = 256
SRC_WARE_ITEMS = 512
SRC_WARE_VEHICLE = 768
SRC_WARE_TANKMAN = 1024
SRC_WARE_KINDS = (SRC_WARE_GOLD,
 SRC_WARE_CREDITS,
 SRC_WARE_ITEMS,
 SRC_WARE_VEHICLE,
 SRC_WARE_TANKMAN)
SRC_WARE_MONEY_KINDS = (SRC_WARE_GOLD, SRC_WARE_CREDITS)
_SRC_WARE_KIND_MASK = 3840
DST_WARE_GOLD = 0
DST_WARE_CREDITS = 4096
DST_WARE_KINDS = (DST_WARE_GOLD, DST_WARE_CREDITS)
_DST_WARE_KIND_MASK = 61440

def makeOfferFlags(offerKind, srcWareKind, dstWareKind, srcEntityType, dstEntityType):
    return offerKind | srcWareKind | dstWareKind | srcEntityType | dstEntityType << 3

ParsedOfferFlags = namedtuple('ParsedOfferFlags', 'offerKind srcWareKind dstWareKind srcEntityType dstEntityType')

def parseOfferFlags(flags):
    raw = (flags & _OFFER_KIND_MASK,
     flags & _SRC_WARE_KIND_MASK,
     flags & _DST_WARE_KIND_MASK,
     flags & 7,
     flags >> 3 & 7)
    return ParsedOfferFlags._make(raw)

def parseSrcEntityTypeFromFlags(flags):
    return flags & 7

def parseDstEntityTypeFromFlags(flags):
    return flags >> 3 & 7

class OutOffers(object):
    Offer = namedtuple('Offer', 'flags dstDBID dstName srcWares dstWares validTill fee')

    def __init__(self, offersDict, outWriterGetter = None):
        offersDict.setdefault('nextID', 0)
        offersDict.setdefault('done', {})
        offersDict.setdefault('out', {})
        self.__data = offersDict
        self.__outWriter = outWriterGetter if outWriterGetter is not None else _WriterGetter(offersDict['out'])
        return

    def __getitem__(self, offerID):
        return _makeOutOffer(self.__data['out'][offerID])

    def get(self, offerID):
        offer = self.__data['out'].get(offerID)
        if offer is not None:
            return _makeOutOffer(offer)
        else:
            return

    def getExt(self, offerID, default = None):
        outExt = self.__data.get('outExt')
        if outExt is None:
            return default
        else:
            return outExt.get(offerID, default)

    def items(self):
        return [ (id, _makeOutOffer(data)) for id, data in self.__data['out'].iteritems() ]

    def clear(self):
        self.__data['out'].clear()
        self.__data['done'].clear()
        self.__data.pop('outExt', None)
        self.__data['nextID'] += 1
        return

    def count(self):
        return len(self.__data['out'])

    def doneOffers(self):
        return self.__data['done']

    def timedOutOffers(self):
        res = []
        currTime = int(time.time())
        for offerID, offer in self.__data['out'].iteritems():
            if offer[5] <= currTime:
                res.append(offerID)

        return res

    def inventorySlots(self):
        vehs = []
        numTmen = 0
        for offer in self.__data['out'].itervalues():
            srcWareKind = offer[0] & _SRC_WARE_KIND_MASK
            if srcWareKind == SRC_WARE_VEHICLE:
                vehs.append(offer[3][0])
            elif srcWareKind == SRC_WARE_TANKMAN:
                numTmen += 1

        return (vehs, numTmen)

    def moveToDone(self, offerID):
        data = self.__data
        data['done'][offerID] = self.__outWriter().pop(offerID)
        outExt = data.get('outExt')
        if outExt is not None:
            outExt.pop(offerID, None)
        data['nextID'] += 1
        return len(data['done'])

    def remove(self, offerID):
        if self.__outWriter().pop(offerID, None) is not None:
            self.__data['nextID'] += 1
            outExt = self.__data.get('outExt')
            if outExt is not None:
                outExt.pop(offerID, None)
        return

    def removeDone(self, offerID):
        self.__data['done'].pop(offerID, None)
        return

    def updateDestination(self, offerID, dstEntityType, dstEntityDBID, dstEntityName):
        raise self.__data['out'][offerID][1] == dstEntityDBID or AssertionError

    def createOffer(self, flags, srcDBID, srcName, dstDBID, dstName, validSec, srcWares, srcFee, dstWares, dstFee, ext = None):
        currTime = int(time.time())
        validTill = currTime + int(validSec)
        offer = (flags,
         dstDBID,
         dstName,
         srcWares,
         dstWares,
         validTill,
         srcFee)
        data = self.__data
        offerID = ((currTime & 1048575) << 12) + (data['nextID'] & 4095)
        data['nextID'] += 1
        if not (offerID not in data['out'] and offerID not in data['done']):
            raise AssertionError
            self.__outWriter()[offerID] = offer
            data.setdefault('outExt', {})[offerID] = ext is not None and ext
        return (offerID, (offerID,
          flags,
          srcDBID,
          srcName,
          srcWares,
          dstWares,
          validTill,
          dstFee))

class InOffers(object):
    Offer = namedtuple('Offer', 'srcOfferID flags srcDBID srcName srcWares dstWares validTill fee')

    def __init__(self, offersDict, inWriterGetter = None):
        offersDict.setdefault('nextID', 0)
        offersDict.setdefault('in', {})
        self.__data = offersDict
        self.__inWriter = inWriterGetter if inWriterGetter is not None else _WriterGetter(offersDict['in'])
        return

    def __getitem__(self, offerID):
        return _makeInOffer(self.__data['in'][offerID])

    def get(self, offerID):
        offer = self.__data['in'].get(offerID)
        if offer is not None:
            return _makeInOffer(offer)
        else:
            return

    def items(self):
        return [ (id, _makeOutOffer(data)) for id, data in self.__data['in'].iteritems() ]

    def clear(self):
        self.__data['in'].clear()
        self.__data['nextID'] += 1

    def count(self):
        return len(self.__data['in'])

    def timedOutOffers(self):
        res = []
        currTime = int(time.time())
        for offerID, offer in self.__data['in'].iteritems():
            if offer[6] <= currTime:
                res.append(offerID)

        return res

    def findOfferBySource(self, srcEntityType, srcEntityDBID, srcOfferID):
        for inOfferID, offer in self.__data['in'].iteritems():
            if offer[0] == srcOfferID and offer[2] == srcEntityDBID and parseSrcEntityTypeFromFlags(offer[1]) == srcEntityType:
                return inOfferID

        return None

    def add(self, offer):
        data = self.__data
        offerID = data['nextID']
        data['nextID'] += 1
        self.__inWriter()[offerID] = tuple(offer)
        return offerID

    def remove(self, offerID):
        if self.__inWriter().pop(offerID, None) is not None:
            self.__data['nextID'] += 1
        return

def collectOutOfferResults(outOffer):
    offerFlags = parseOfferFlags(outOffer.flags)
    gold = 0
    credits = 0
    items = None
    if offerFlags.srcWareKind == SRC_WARE_GOLD:
        gold -= outOffer.srcWares + outOffer.fee
    elif offerFlags.srcWareKind == SRC_WARE_CREDITS:
        credits -= outOffer.srcWares + outOffer.fee
    else:
        items = outOffer.srcWares
    if offerFlags.dstWareKind == DST_WARE_GOLD:
        gold += outOffer.dstWares
    else:
        credits += outOffer.dstWares
    return (offerFlags,
     gold,
     credits,
     items)

def collectInOfferResults(inOffer):
    offerFlags = parseOfferFlags(inOffer.flags)
    gold = 0
    credits = 0
    items = None
    if offerFlags.srcWareKind == SRC_WARE_GOLD:
        gold += inOffer.srcWares
    elif offerFlags.srcWareKind == SRC_WARE_CREDITS:
        credits += inOffer.srcWares
    else:
        items = inOffer.srcWares
    if offerFlags.dstWareKind == DST_WARE_GOLD:
        gold -= inOffer.dstWares + inOffer.fee
    else:
        credits -= inOffer.dstWares + inOffer.fee
    return (offerFlags,
     gold,
     credits,
     items)

_makeOutOffer = OutOffers.Offer._make
_makeInOffer = InOffers.Offer._make

class _WriterGetter(object):

    def __init__(self, dict):
        self.__d = dict

    def __call__(self):
        return self.__d


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
import pydot
import tensorflow as tf
import keras
import seaborn as sb
from sklearn.metrics import confusion_matrix

train = pd.read_csv('fashion-mnist_train.csv')
test = pd.read_csv('fashion-mnist_test.csv')

# Check if there is any null value in dataset
df = pd.concat([train,test], ignore_index=True)
df.isnull().sum().sort_values(ascending=True)

X_train = train.drop('label',axis = 1)
y_train = train['label']
X_test = test.drop('label',axis = 1)
y_test = test['label']
X_train.shape

#Each example is of 28*28 grayscale image. Many NN accept the image of fixed size
# so we have to resize the image
x_train_reshape = X_train.values.reshape(-1,28,28)
x_test_reshape = X_test.values.reshape(-1,28,28)
x_train_reshape.shape

# There are 10 labels in data set
col_names = ['T-shirt/top','Trouser','Pullover','Dress','Coat','Sandal','Shirt','Sneaker','Bag','Ankle boot']

# We can print the name of labels
print(col_names[y_train[0]])

# Perform Normalization
x_train=X_train/255
x_test=X_test/255

# Implementing sequential model from tensorflow Deep Learning Library
model = tf.keras.models.Sequential()
model.add(keras.layers.Flatten(input_shape=[28,28]))
model.add(keras.layers.Dense(units=128, activation='relu', input_shape=(784, )))
model.add(keras.layers.Dense(units=24, activation='relu'))
model.add(keras.layers.Dropout(0.2))
model.add(keras.layers.Dense(units=10, activation='softmax'))
# Compile the model
model.compile(optimizer='adam',loss='sparse_categorical_crossentropy',metrics=['accuracy'])
print(model.summary())

# Train our model with 30 epoch using fit method
model_history = model.fit(x_train,y_train,epochs = 30)
print(model_history.params)