zero cardinality values

multi_label_precision_recall_accuracy_fscore.py

@@ -69,6 +69,7 @@ Examples:
         "accuracy": 1.0,
         "fscore": 1.0
     }
+    
 """
 
 
@@ -81,6 +82,8 @@ class MultiLabelPrecisionRecallAccuracyFscore(evaluate.Metric):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.beta = kwargs.get("beta", 1.0)
+        self.zero_cardinality_precision = kwargs.get("zero_cardinality_precision", 0.0)  # default value for precision when prediction is empty, when precision and recall are both 0, it is always 1
+        self.zero_cardinality_recall = kwargs.get("zero_cardinality_recall", 0.0)   # default value for recall when reference is empty, when precision and recall are both 0, it is always 1
         self.use_multiset = self.config_name == "multiset"
 
     def _info(self):
@@ -126,9 +129,9 @@ class MultiLabelPrecisionRecallAccuracyFscore(evaluate.Metric):
             prediction_cardinality = len(prediction)
             reference_cardinality = len(reference)
 
-        precision = intersection_cardinality / prediction_cardinality if prediction_cardinality > 0 else 0
-        recall = intersection_cardinality / reference_cardinality if reference_cardinality > 0 else 0
-        accuracy = intersection_cardinality / union_cardinality if union_cardinality > 0 else 0
+        precision = intersection_cardinality / prediction_cardinality if prediction_cardinality > 0 else self.zero_cardinality_precision
+        recall = intersection_cardinality / reference_cardinality if reference_cardinality > 0 else self.zero_cardinality_recall
+        accuracy = intersection_cardinality / union_cardinality  # no need for check, as union_cardinality is always > 0 if prediction and reference are not empty
 
         return precision, recall, accuracy
 

tests.py

@@ -8,6 +8,7 @@ class MultiLabelPrecisionRecallAccuracyFscoreTest(TestCase):
     All of these tests are also used for multiset configuration. So please mind this and write the test in a way that
     it is valid for both configurations (do not use same label multiple times).
     """
+
     def setUp(self):
         self.multi_label_precision_recall_accuracy_fscore = MultiLabelPrecisionRecallAccuracyFscore()
 
@@ -149,7 +150,7 @@ class MultiLabelPrecisionRecallAccuracyFscoreTest(TestCase):
                 "precision": 1.0,
                 "recall": 0.5,
                 "accuracy": 0.5,
-                "fscore": 2/3
+                "fscore": 2 / 3
             },
             self.multi_label_precision_recall_accuracy_fscore.compute(
                 predictions=[
@@ -167,7 +168,7 @@ class MultiLabelPrecisionRecallAccuracyFscoreTest(TestCase):
                 "precision": 0.5,
                 "recall": 1.0,
                 "accuracy": 0.5,
-                "fscore": 2/3
+                "fscore": 2 / 3
             },
             self.multi_label_precision_recall_accuracy_fscore.compute(
                 predictions=[
@@ -184,7 +185,7 @@ class MultiLabelPrecisionRecallAccuracyFscoreTest(TestCase):
             {
                 "precision": 0.5,
                 "recall": 0.5,
-                "accuracy": 1/3,
+                "accuracy": 1 / 3,
                 "fscore": 0.5
             },
             self.multi_label_precision_recall_accuracy_fscore.compute(
@@ -200,10 +201,10 @@ class MultiLabelPrecisionRecallAccuracyFscoreTest(TestCase):
     def test_partial_match_multi_sample(self):
         self.assertDictEqual(
             {
-                "precision": 2.5/3,
-                "recall": 2/3,
+                "precision": 2.5 / 3,
+                "recall": 2 / 3,
                 "accuracy": 0.5,
-                "fscore": 2*(2.5/3 * 2/3) / (2.5/3 + 2/3)
+                "fscore": 2 * (2.5 / 3 * 2 / 3) / (2.5 / 3 + 2 / 3)
             },
             self.multi_label_precision_recall_accuracy_fscore.compute(
                 predictions=[
@@ -223,10 +224,10 @@ class MultiLabelPrecisionRecallAccuracyFscoreTest(TestCase):
         self.multi_label_precision_recall_accuracy_fscore.beta = 2
         self.assertDictEqual(
             {
-                "precision": 2.5/3,
-                "recall": 2/3,
+                "precision": 2.5 / 3,
+                "recall": 2 / 3,
                 "accuracy": 0.5,
-                "fscore": 5*(2.5/3 * 2/3) / (4*2.5/3 + 2/3)
+                "fscore": 5 * (2.5 / 3 * 2 / 3) / (4 * 2.5 / 3 + 2 / 3)
             },
             self.multi_label_precision_recall_accuracy_fscore.compute(
                 predictions=[
@@ -266,7 +267,8 @@ class MultiLabelPrecisionRecallAccuracyFscoreTest(TestCase):
 
 class MultiLabelPrecisionRecallAccuracyFscoreTestMultiset(MultiLabelPrecisionRecallAccuracyFscoreTest):
     def setUp(self):
-        self.multi_label_precision_recall_accuracy_fscore = MultiLabelPrecisionRecallAccuracyFscore(config_name="multiset")
+        self.multi_label_precision_recall_accuracy_fscore = MultiLabelPrecisionRecallAccuracyFscore(
+            config_name="multiset")
 
     def test_multiset_eok(self):
         self.assertDictEqual(
@@ -291,13 +293,12 @@ class MultiLabelPrecisionRecallAccuracyFscoreTestMultiset(MultiLabelPrecisionRec
         )
 
     def test_multiset_partial_match(self):
-
         self.assertDictEqual(
             {
                 "precision": 1.0,
                 "recall": 0.5,
                 "accuracy": 0.5,
-                "fscore": 2/3
+                "fscore": 2 / 3
             },
             self.multi_label_precision_recall_accuracy_fscore.compute(
                 predictions=[
@@ -310,15 +311,15 @@ class MultiLabelPrecisionRecallAccuracyFscoreTestMultiset(MultiLabelPrecisionRec
         )
 
     def test_multiset_partial_match_multi_sample(self):
-        p = (1+2/3) / 2
-        r = (3/4 + 1) / 2
+        p = (1 + 2 / 3) / 2
+        r = (3 / 4 + 1) / 2
 
         self.assertDictEqual(
             {
                 "precision": p,
                 "recall": r,
-                "accuracy": (3/4 + 2/3) / 2,
-                "fscore": 2*p*r / (p + r)
+                "accuracy": (3 / 4 + 2 / 3) / 2,
+                "fscore": 2 * p * r / (p + r)
             },
             self.multi_label_precision_recall_accuracy_fscore.compute(
                 predictions=[
@@ -331,3 +332,74 @@ class MultiLabelPrecisionRecallAccuracyFscoreTestMultiset(MultiLabelPrecisionRec
                 ]
             )
         )
+
+    def test_zero_cardinality_precision(self):
+        self.multi_label_precision_recall_accuracy_fscore.zero_cardinality_precision = 0.5
+        self.assertEqual(0.5,
+                         self.multi_label_precision_recall_accuracy_fscore.compute(
+                             predictions=[
+                                 []
+                             ],
+                             references=[
+                                 [0, 1, 1],
+                             ]
+                         )["precision"]
+                         )
+
+        self.assertEqual(1.0,
+                         self.multi_label_precision_recall_accuracy_fscore.compute(
+                             predictions=[
+                                 []
+                             ],
+                             references=[
+                                 [],
+                             ]
+                         )["precision"]
+                         )
+
+        self.assertEqual(2 / 3,
+                         self.multi_label_precision_recall_accuracy_fscore.compute(
+                             predictions=[
+                                 [1, 2, 3]
+                             ],
+                             references=[
+                                 [1, 2],
+                             ]
+                         )["precision"]
+                         )
+
+    def test_zero_cardinality_recall(self):
+        self.multi_label_precision_recall_accuracy_fscore.zero_cardinality_recall = 0.5
+        self.assertEqual(0.5,
+                         self.multi_label_precision_recall_accuracy_fscore.compute(
+                             predictions=[
+                                 [0, 1, 1],
+                             ],
+                             references=[
+                                 []
+                             ]
+                         )["recall"]
+                         )
+
+        self.assertEqual(1.0,
+                         self.multi_label_precision_recall_accuracy_fscore.compute(
+                             predictions=[
+                                 [],
+                             ],
+                             references=[
+                                 [],
+                             ]
+                         )["recall"]
+                         )
+
+        self.assertEqual(2 / 3,
+                         self.multi_label_precision_recall_accuracy_fscore.compute(
+                             predictions=[
+                                 [1, 2],
+                             ],
+                             references=[
+                                 [1, 2, 3]
+                             ]
+                         )["recall"]
+                         )
+