adding wil metric

README.md

@@ -1,5 +1,5 @@
 ---
-title: Wil
+title: WIL
 emoji: 🐠
 colorFrom: red
 colorTo: purple
@@ -7,6 +7,29 @@ sdk: gradio
 sdk_version: 3.27.0
 app_file: app.py
 pinned: false
+tags:
+  - evaluate
+  - metric
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Word Information Loss can be used to evaluate the performance of an automatic speech recognizer. It has information-theoretic backings, is symmetric between predictions and targets, and is bounded between 0 and 1.
+
+The formula for WIL is
+
+WIL = 1 - (C/P)(C/T)
+
+where
+C is the number of correct words,
+P is the number of words in the prediction,
+T is the number of words in the target.
+
+This value measures the amount of information loss between two sentences. A score of 0 indicates that the prediction and target match perfectly.
+
+Here is a comparison of WER and WIL: (assuming that X,Y,Z each represents a different word)
+| Target | Prediction | WER | WIL |
+| ------------- | ------------- | ------------- | ------------- | 
+| X  | X | 1 | 1 |
+| X  | Y | 0 | 0 |
+| X  | XZZZ | 3 | 0.75 |
+| XYYY | X | 0.75 | 0.75 |
+| XYY | XZ | 0.67 | 0.83 |

app.py

@@ -0,0 +1,6 @@
+import evaluate
+from evaluate.utils import launch_gradio_widget
+
+
+module = evaluate.load("wil")
+launch_gradio_widget(module)

requirements.txt

@@ -0,0 +1,2 @@
+git+https://github.com/huggingface/evaluate@0ca575d7aa0764ea646dcd5a27cb952e587ce9eb
+jiwer

wil.py

@@ -0,0 +1,92 @@
+# Copyright 2021 The HuggingFace Evaluate Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Word Information Loss (WIL) metric. """
+
+import datasets
+from jiwer import process_words
+
+import evaluate
+
+
+_CITATION = """\
+@inproceedings{inproceedings,
+    author = {Morris, Andrew and Maier, Viktoria and Green, Phil},
+    year = {2004},
+    month = {01},
+    pages = {},
+    title = {From WER and RIL to MER and WIL: improved evaluation measures for connected speech recognition.}
+}y
+"""
+
+_DESCRIPTION = """\
+Word Information Loss can be used to evaluate the performance of an automatic speech recognizer. It has information-theoretic backings, is symmetric between predictions and targets, and is bounded between 0 and 1.
+The formula for WIL is
+WIL = 1 - (C/P)(C/T)
+where
+C is the number of correct words,
+P is the number of words in the prediction,
+T is the number of words in the target.
+
+This value measures the amount of information loss between two sentences. A score of 0 indicates that the prediction and target match perfectly.
+"""
+
+_KWARGS_DESCRIPTION = """
+Compute the WIL between two sets of words.
+Args:
+    targets: List of target words.
+    predictions: List of transcriptions to evaluate.
+    concatenate_texts (bool, default=False): Whether to concatenate the WIL of the concanated strings or the mean WIL for each pair.
+Returns:
+    (float): the word information loss
+Examples:
+    >>> predictions = ["this is a prediction", "there is an other sample"]
+    >>> targets = ["this is the target", "there is another one"]
+    >>> wil = evaluate.load("wil")
+    >>> wil_score = wil.compute(predictions=predictions, targets=targets)
+    >>> print(wil_score)
+    0.775
+    >>> wil_score = wil.compute(predictions=targets, targets=predictions)
+    >>> print(wil_score)
+    0.775
+"""
+
+
+@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
+class WIL(evaluate.Metric):
+    def _info(self):
+        return evaluate.MetricInfo(
+            description=_DESCRIPTION,
+            citation=_CITATION,
+            inputs_description=_KWARGS_DESCRIPTION,
+            features=datasets.Features(
+                {
+                    "predictions": datasets.Value("string", id="sequence"),
+                    "references": datasets.Value("string", id="sequence"),
+                }
+            ),
+            codebase_urls=["https://github.com/jitsi/jiwer/"],
+            reference_urls=[
+                "https://en.wikipedia.org/wiki/Word_error_rate",
+            ],
+        )
+
+    def _compute(self, predictions=None, references=None, concatenate_texts=False):
+        if concatenate_texts:
+            return process_words(references, predictions).wil
+        else:
+            total = 0
+            for prediction, reference in zip(predictions, references):
+                measures = process_words(reference, prediction).wil
+                total += measures
+            return total/len(predictions)