Document multi-measure Perf support

CompStats/bootstrap.py

@@ -53,7 +53,7 @@ def __init__(self,
         self.statistic = statistic
         self.num_samples = num_samples
         self.n_jobs = n_jobs
-        self.BiB = BiB  # Guardar el parámetro BiB        
+        self.BiB = BiB  # Store the BiB parameter
         self._samples = None
         self._calls = {}
         self._info = {}
@@ -72,7 +72,7 @@ def get_params(self):
         return dict(statistic=self.statistic,
                     num_samples=self.num_samples,
                     n_jobs=self.n_jobs,
-                    BiB=self.BiB)  # Añadir BiB a los parámetros
+                    BiB=self.BiB)  # Add BiB to the parameters
 
     def __sklearn_clone__(self):
         klass = self.__class__

CompStats/utils.py

@@ -20,32 +20,56 @@
 
 
 def progress_bar(arg, use_tqdm: bool=True, **kwargs):
-    """Progress bar using tqdm"""
+    """Wrap `arg` in a :py:class:`tqdm.tqdm` progress bar.
+
+    Returns `arg` unchanged when tqdm is not installed or :py:attr:`use_tqdm` is
+    `False`, so callers can iterate over the result the same way regardless of
+    whether a progress bar is actually shown.
+
+    :param arg: Iterable to wrap.
+    :param use_tqdm: Whether to show the progress bar, default=True.
+    :type use_tqdm: bool
+    :param kwargs: Extra keyword arguments passed to :py:class:`tqdm.tqdm`.
+    :return: `arg`, optionally wrapped in a :py:class:`tqdm.tqdm` iterator.
+    """
     if not USE_TQDM or not use_tqdm:
         return arg
     return tqdm(arg, **kwargs)
 
 
 def metrics_docs(hy_name='y_pred', attr_name='score_func'):
-    """Decorator to set docs"""
+    """Decorator that injects the shared :py:class:`~CompStats.interface.Perf`
+    docstring into a :py:mod:`CompStats.metrics` wrapper (e.g.
+    :py:func:`~CompStats.metrics.f1_score`).
+
+    :param hy_name: Name used for the predictions parameter in the generated
+        docstring (e.g. ``y_pred`` or ``y_score``, matching the wrapped
+        :py:mod:`sklearn.metrics` function's own parameter name).
+    :type hy_name: str
+    :param attr_name: Which :py:class:`~CompStats.interface.Perf` argument the
+        wrapped function's measure is passed as, ``score_func`` or ``error_func``.
+    :type attr_name: str
+    """
 
     def perf_docs(func):
         """Decorator to Perf to write :py:class:`~sklearn.metrics` documentation"""
 
         func.__doc__ = f""":py:class:`~CompStats.interface.Perf` with :py:func:`~sklearn.metrics.{func.__name__}` as :py:attr:`{attr_name}.` The parameters not described can be found in :py:func:`~sklearn.metrics.{func.__name__}`.
-
-    :param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement. 
-    :type y_true: numpy.ndarray or pandas.DataFrame 
-    :param {hy_name}: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`y_pred.` 
-    :type {hy_name}: numpy.ndarray 
-    :param kwargs: Predictions, the algorithms will be identified using the keyword  
-    :type kwargs: numpy.ndarray 
-    :param num_samples: Number of bootstrap samples, default=500. 
-    :type num_samples: int 
-    :param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads. 
-    :type n_jobs: int 
-    :param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True 
-    :type use_tqdm: bool 
+
+    :param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement.
+    :type y_true: numpy.ndarray or pandas.DataFrame
+    :param {hy_name}: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`y_pred.`
+    :type {hy_name}: numpy.ndarray
+    :param kwargs: Predictions, the algorithms will be identified using the keyword
+    :type kwargs: numpy.ndarray
+    :param num_samples: Number of bootstrap samples, default=500.
+    :type num_samples: int
+    :param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads.
+    :type n_jobs: int
+    :param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True
+    :type use_tqdm: bool
+
+    :py:func:`~CompStats.metrics.{func.__name__}.measure` builds the tagged callable used internally as :py:attr:`{attr_name}`; call it directly (e.g. ``{func.__name__}.measure(...)``) to combine this metric with others into a single, multi-measure :py:class:`~CompStats.interface.Perf` -- see :py:class:`~CompStats.interface.Perf`'s class docstring for a worked example.
 
     """ + func.__doc__
 
@@ -61,7 +85,27 @@ def dataframe(instance, value_name:str='Score',
               var_name:str='Performance',
               alg_legend:str='Algorithm',
               perf_names:list=None):
-    """Dataframe"""
+    """Melt a :py:class:`~CompStats.interface.Perf` or
+    :py:class:`~CompStats.interface.Difference` instance's bootstrap samples into
+    a long-format :py:class:`pandas.DataFrame`, ready for seaborn's ``catplot``
+    (used by :py:meth:`~CompStats.interface.Perf.plot` and
+    :py:meth:`~CompStats.interface.Difference.plot`).
+
+    :param instance: Instance holding the bootstrap samples to melt.
+    :type instance: CompStats.interface.Perf or CompStats.interface.Difference
+    :param value_name: Column name for the statistic's value.
+    :type value_name: str
+    :param var_name: Column name identifying which measure a row belongs to,
+        only used when `instance` holds more than one measure.
+    :type var_name: str
+    :param alg_legend: Column name identifying which algorithm a row belongs to.
+    :type alg_legend: str
+    :param perf_names: Display name for each measure, only used when `instance`
+        holds more than one measure.
+    :type perf_names: list
+    :return: Long-format dataframe with one row per bootstrap sample.
+    :rtype: pandas.DataFrame
+    """
     import pandas as pd
     statistic = instance.statistic
     if not isinstance(statistic, dict):

docs/source/metrics_api.rst

@@ -96,11 +96,61 @@ difference p-values  w.r.t Hist. Grad. Boost. Tree
 0.0100 <= alg-1
 0.3240 <= Random Forest
 
-The class :py:class:`~CompStats.Difference` has the :py:class:`~CompStats.Difference.plot` method that can be used to depict the difference with respectto the best. 
+The class :py:class:`~CompStats.Difference` has the :py:class:`~CompStats.Difference.plot` method that can be used to depict the difference with respectto the best.
 
 >>> diff.plot()
 
 .. image:: digits_difference.png
 
+Multi-measure Perf
+--------------------
+
+A single competition can also be evaluated with more than one measure at once (e.g., macro-F1 together with macro-recall) by passing a list of functions to :py:attr:`score_func`/:py:attr:`error_func`. :py:attr:`score_func` and :py:attr:`error_func` can even be combined to mix score-type and error-type measures, with different Bigger-is-Better (BiB) directions, into a single :py:class:`~CompStats.interface.Perf` instance. Every measure is evaluated on the same bootstrap resamples, so comparisons across algorithms remain paired for each measure.
+
+Every :py:mod:`CompStats.metrics` wrapper exposes a ``.measure`` factory (e.g. :py:func:`~CompStats.metrics.f1_score.measure`) that builds the tagged callable used internally as :py:attr:`score_func`/:py:attr:`error_func`; call it directly to compose several measures, as shown next.
+
+>>> from CompStats.interface import Perf
+>>> from CompStats.metrics import f1_score, recall_score
+>>> mperf = Perf(y_val, hy, forest=ens.predict(X_val),
+...              score_func=[f1_score.measure(average='macro'),
+...                          recall_score.measure(average='macro')],
+...              measure_names=['macro-F1', 'macro-Recall'])
+>>> mperf
+<Perf(score_func=macro-F1+macro-Recall)>
+Statistic with its standard error (se)
+statistic (se)
+0.9783 (0.0061), 0.9786 (0.0060) <= forest
+0.9440 (0.0098), 0.9442 (0.0098) <= alg-1
+
+:py:attr:`measure_names` is optional; when omitted, each measure is labeled with its function's ``__name__`` (e.g. ``f1_score``, ``recall_score``). The properties :py:func:`~CompStats.interface.Perf.statistic`, :py:func:`~CompStats.interface.Perf.se`, and :py:func:`~CompStats.interface.Perf.ci` return one value per measure for every system.
+
+>>> mperf.statistic
+{'forest': array([0.97828319, 0.97855524]), 'alg-1': array([0.94399193, 0.94424915])}
+>>> mperf.se
+{'forest': array([0.00605715, 0.00595743]), 'alg-1': array([0.0098302 , 0.00978099])}
+>>> mperf.ci
+{'alg-1': (array([0.92393337, 0.92473771]), array([0.96193002, 0.96198806])), 'forest': (array([0.96618741, 0.9666813 ]), array([0.98902657, 0.98936809]))}
+
+:py:func:`~CompStats.interface.Perf.plot` facets the resulting figure by measure, and :py:func:`~CompStats.interface.Perf.difference` reports one p-value per measure for each system compared to the best.
+
+>>> mperf.plot()
+>>> mperf.difference()
+<Difference>
+difference p-values
+forest, forest <= Best
+0.0000, 0.0000 <= alg-1
+1.0000, 1.0000 <= forest
+
+The convenience wrappers :py:func:`~CompStats.metrics.macro_f1`, :py:func:`~CompStats.metrics.macro_recall`, and :py:func:`~CompStats.metrics.macro_precision` are ready-made, multi-measure-friendly shortcuts for macro-averaged F1, recall, and precision; each also exposes its own ``.measure`` factory (e.g. :py:func:`~CompStats.metrics.macro_f1.measure`), so they can be combined the same way as any other :py:mod:`CompStats.metrics` wrapper.
+
+>>> from CompStats.metrics import macro_f1, macro_recall
+>>> Perf(y_val, hy, forest=ens.predict(X_val),
+...      score_func=[macro_f1.measure(), macro_recall.measure()])
+<Perf(score_func=f1_score+recall_score)>
+Statistic with its standard error (se)
+statistic (se)
+0.9783 (0.0061), 0.9786 (0.0060) <= forest
+0.9440 (0.0101), 0.9442 (0.0101) <= alg-1
+
 .. automodule:: CompStats.metrics
    :members:

quarto/CompStats.qmd

@@ -110,4 +110,62 @@ hist = HistGradientBoostingClassifier().fit(X_train, y_train)
 _ = score(hist.predict(X_val), name='Hist. Grad. Boost. Tree')
 score.plot()
 ```
+:::
+
+# Multi-measure Support
+
+## Column
+
+Sometimes a competition needs to be evaluated with more than one measure at once, e.g., macro-F1 together with macro-recall. `CompStats` supports this by passing a list of functions to `score_func`/`error_func`; the two can even be combined to mix score-type and error-type measures, with different Bigger-is-Better directions, into a single `Perf` instance. Every measure is evaluated on the same bootstrap resamples, so comparisons across algorithms remain paired for each measure.
+
+Every `CompStats.metrics` wrapper exposes a `.measure` factory (e.g. `f1_score.measure`) that builds the tagged callable used internally as `score_func`/`error_func`; call it directly to compose several measures, as shown next.
+
+::: {.card title="Combining two measures" .flow}
+```{python}
+#| echo: true
+
+from CompStats.interface import Perf
+from CompStats.metrics import f1_score, recall_score
+
+mperf = Perf(y_val, hy, forest=ens.predict(X_val),
+             score_func=[f1_score.measure(average='macro'),
+                         recall_score.measure(average='macro')],
+             measure_names=['macro-F1', 'macro-Recall'])
+mperf
+```
+:::
+
+`measure_names` is optional; when omitted, each measure is labeled with its function's `__name__`. The properties `statistic`, `se`, and `ci` return one value per measure for every system.
+
+::: {.card title="Inspecting the measures" .flow}
+```{python}
+#| echo: true
+
+mperf.statistic
+```
+:::
+
+`plot` facets the resulting figure by measure, and `difference` reports one p-value per measure for each system compared to the best.
+
+::: {.card title="Difference across measures" .flow}
+```{python}
+#| echo: true
+
+mperf.difference()
+```
+:::
+
+## Column
+
+The convenience wrappers `macro_f1`, `macro_recall`, and `macro_precision` are ready-made, multi-measure-friendly shortcuts for macro-averaged F1, recall, and precision; each also exposes its own `.measure` factory, so they can be combined the same way as any other `CompStats.metrics` wrapper.
+
+::: {.card title="Convenience wrappers" .flow}
+```{python}
+#| echo: true
+
+from CompStats.metrics import macro_f1, macro_recall
+
+Perf(y_val, hy, forest=ens.predict(X_val),
+     score_func=[macro_f1.measure(), macro_recall.measure()])
+```
 :::