Update Space (evaluate main: eaf34a7d)

README.md

@@ -42,10 +42,20 @@ At minimum, this metric takes as input a list of predictions and a list of refer
 {'rouge1': 1.0, 'rouge2': 1.0, 'rougeL': 1.0, 'rougeLsum': 1.0}
 ```
 
+It can also deal with lists of references for each predictions:
+```python
+>>> rouge = evaluate.load('rouge')
+>>> predictions = ["hello there", "general kenobi"]
+>>> references = [["hello", "there"], ["general kenobi", "general yoda"]]
+>>> results = rouge.compute(predictions=predictions,
+...                         references=references)
+>>> print(results)
+{'rouge1': 0.8333, 'rouge2': 0.5, 'rougeL': 0.8333, 'rougeLsum': 0.8333}```
+
 ### Inputs
 - **predictions** (`list`): list of predictions to score. Each prediction
         should be a string with tokens separated by spaces.
-- **references** (`list`): list of reference for each prediction. Each
+- **references** (`list` or `list[list]`): list of reference for each prediction or a list of several references per prediction. Each
         reference should be a string with tokens separated by spaces.
 - **rouge_types** (`list`): A list of rouge types to calculate. Defaults to `['rouge1', 'rouge2', 'rougeL', 'rougeLsum']`.
     - Valid rouge types:

requirements.txt

@@ -2,4 +2,4 @@ git+https://github.com/huggingface/evaluate@a45df1eb9996eec64ec3282ebe554061cb36
 datasets~=2.0
 absl-py
 nltk
-rouge_score
+rouge_score>=0.1.2

rouge.py

@@ -87,12 +87,20 @@ class Rouge(evaluate.Metric):
             description=_DESCRIPTION,
             citation=_CITATION,
             inputs_description=_KWARGS_DESCRIPTION,
-            features=datasets.Features(
-                {
-                    "predictions": datasets.Value("string", id="sequence"),
-                    "references": datasets.Value("string", id="sequence"),
-                }
-            ),
+            features=[
+                datasets.Features(
+                    {
+                        "predictions": datasets.Value("string", id="sequence"),
+                        "references": datasets.Sequence(datasets.Value("string", id="sequence")),
+                    }
+                ),
+                datasets.Features(
+                    {
+                        "predictions": datasets.Value("string", id="sequence"),
+                        "references": datasets.Value("string", id="sequence"),
+                    }
+                ),
+            ],
             codebase_urls=["https://github.com/google-research/google-research/tree/master/rouge"],
             reference_urls=[
                 "https://en.wikipedia.org/wiki/ROUGE_(metric)",
@@ -104,6 +112,8 @@ class Rouge(evaluate.Metric):
         if rouge_types is None:
             rouge_types = ["rouge1", "rouge2", "rougeL", "rougeLsum"]
 
+        multi_ref = isinstance(references[0], list)
+
         scorer = rouge_scorer.RougeScorer(rouge_types=rouge_types, use_stemmer=use_stemmer)
         if use_aggregator:
             aggregator = scoring.BootstrapAggregator()
@@ -111,7 +121,10 @@ class Rouge(evaluate.Metric):
             scores = []
 
         for ref, pred in zip(references, predictions):
-            score = scorer.score(ref, pred)
+            if multi_ref:
+                score = scorer.score_multi(ref, pred)
+            else:
+                score = scorer.score(ref, pred)
             if use_aggregator:
                 aggregator.add_scores(score)
             else: