From d87c54457c480035a6312bfdee00d1e07e42a67e Mon Sep 17 00:00:00 2001 From: Mario Graff Date: Sat, 22 Aug 2026 13:54:21 +0000 Subject: [PATCH] Document multi-measure Perf support and fix documentation gaps (closes #32) Documents #30's multi-measure Perf/Difference support, which had tests and inline docstrings but no user-facing documentation: metrics_docs() now explains .measure(), utils.py helpers get full docstrings, the two stray Spanish comments in bootstrap.py are translated, and both metrics_api.rst and CompStats.qmd gain a worked multi-measure walkthrough plus a mention of the macro_f1/macro_recall/macro_precision convenience wrappers. Co-Authored-By: Claude Sonnet 5 --- CompStats/bootstrap.py | 4 +- CompStats/utils.py | 76 +++++++++++++++++++++++++++++-------- docs/source/metrics_api.rst | 52 ++++++++++++++++++++++++- quarto/CompStats.qmd | 58 ++++++++++++++++++++++++++++ 4 files changed, 171 insertions(+), 19 deletions(-) diff --git a/CompStats/bootstrap.py b/CompStats/bootstrap.py index 791682a..beb2ec3 100644 --- a/CompStats/bootstrap.py +++ b/CompStats/bootstrap.py @@ -53,7 +53,7 @@ def __init__(self, self.statistic = statistic self.num_samples = num_samples self.n_jobs = n_jobs - self.BiB = BiB # Guardar el parámetro BiB + self.BiB = BiB # Store the BiB parameter self._samples = None self._calls = {} self._info = {} @@ -72,7 +72,7 @@ def get_params(self): return dict(statistic=self.statistic, num_samples=self.num_samples, n_jobs=self.n_jobs, - BiB=self.BiB) # Añadir BiB a los parámetros + BiB=self.BiB) # Add BiB to the parameters def __sklearn_clone__(self): klass = self.__class__ diff --git a/CompStats/utils.py b/CompStats/utils.py index 85d8ef0..735c981 100644 --- a/CompStats/utils.py +++ b/CompStats/utils.py @@ -20,32 +20,56 @@ def progress_bar(arg, use_tqdm: bool=True, **kwargs): - """Progress bar using tqdm""" + """Wrap `arg` in a :py:class:`tqdm.tqdm` progress bar. + + Returns `arg` unchanged when tqdm is not installed or :py:attr:`use_tqdm` is + `False`, so callers can iterate over the result the same way regardless of + whether a progress bar is actually shown. + + :param arg: Iterable to wrap. + :param use_tqdm: Whether to show the progress bar, default=True. + :type use_tqdm: bool + :param kwargs: Extra keyword arguments passed to :py:class:`tqdm.tqdm`. + :return: `arg`, optionally wrapped in a :py:class:`tqdm.tqdm` iterator. + """ if not USE_TQDM or not use_tqdm: return arg return tqdm(arg, **kwargs) def metrics_docs(hy_name='y_pred', attr_name='score_func'): - """Decorator to set docs""" + """Decorator that injects the shared :py:class:`~CompStats.interface.Perf` + docstring into a :py:mod:`CompStats.metrics` wrapper (e.g. + :py:func:`~CompStats.metrics.f1_score`). + + :param hy_name: Name used for the predictions parameter in the generated + docstring (e.g. ``y_pred`` or ``y_score``, matching the wrapped + :py:mod:`sklearn.metrics` function's own parameter name). + :type hy_name: str + :param attr_name: Which :py:class:`~CompStats.interface.Perf` argument the + wrapped function's measure is passed as, ``score_func`` or ``error_func``. + :type attr_name: str + """ def perf_docs(func): """Decorator to Perf to write :py:class:`~sklearn.metrics` documentation""" func.__doc__ = f""":py:class:`~CompStats.interface.Perf` with :py:func:`~sklearn.metrics.{func.__name__}` as :py:attr:`{attr_name}.` The parameters not described can be found in :py:func:`~sklearn.metrics.{func.__name__}`. - - :param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement. - :type y_true: numpy.ndarray or pandas.DataFrame - :param {hy_name}: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`y_pred.` - :type {hy_name}: numpy.ndarray - :param kwargs: Predictions, the algorithms will be identified using the keyword - :type kwargs: numpy.ndarray - :param num_samples: Number of bootstrap samples, default=500. - :type num_samples: int - :param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads. - :type n_jobs: int - :param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True - :type use_tqdm: bool + + :param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement. + :type y_true: numpy.ndarray or pandas.DataFrame + :param {hy_name}: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`y_pred.` + :type {hy_name}: numpy.ndarray + :param kwargs: Predictions, the algorithms will be identified using the keyword + :type kwargs: numpy.ndarray + :param num_samples: Number of bootstrap samples, default=500. + :type num_samples: int + :param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads. + :type n_jobs: int + :param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True + :type use_tqdm: bool + + :py:func:`~CompStats.metrics.{func.__name__}.measure` builds the tagged callable used internally as :py:attr:`{attr_name}`; call it directly (e.g. ``{func.__name__}.measure(...)``) to combine this metric with others into a single, multi-measure :py:class:`~CompStats.interface.Perf` -- see :py:class:`~CompStats.interface.Perf`'s class docstring for a worked example. """ + func.__doc__ @@ -61,7 +85,27 @@ def dataframe(instance, value_name:str='Score', var_name:str='Performance', alg_legend:str='Algorithm', perf_names:list=None): - """Dataframe""" + """Melt a :py:class:`~CompStats.interface.Perf` or + :py:class:`~CompStats.interface.Difference` instance's bootstrap samples into + a long-format :py:class:`pandas.DataFrame`, ready for seaborn's ``catplot`` + (used by :py:meth:`~CompStats.interface.Perf.plot` and + :py:meth:`~CompStats.interface.Difference.plot`). + + :param instance: Instance holding the bootstrap samples to melt. + :type instance: CompStats.interface.Perf or CompStats.interface.Difference + :param value_name: Column name for the statistic's value. + :type value_name: str + :param var_name: Column name identifying which measure a row belongs to, + only used when `instance` holds more than one measure. + :type var_name: str + :param alg_legend: Column name identifying which algorithm a row belongs to. + :type alg_legend: str + :param perf_names: Display name for each measure, only used when `instance` + holds more than one measure. + :type perf_names: list + :return: Long-format dataframe with one row per bootstrap sample. + :rtype: pandas.DataFrame + """ import pandas as pd statistic = instance.statistic if not isinstance(statistic, dict): diff --git a/docs/source/metrics_api.rst b/docs/source/metrics_api.rst index 4a600a6..0f0e8d5 100644 --- a/docs/source/metrics_api.rst +++ b/docs/source/metrics_api.rst @@ -96,11 +96,61 @@ difference p-values w.r.t Hist. Grad. Boost. Tree 0.0100 <= alg-1 0.3240 <= Random Forest -The class :py:class:`~CompStats.Difference` has the :py:class:`~CompStats.Difference.plot` method that can be used to depict the difference with respectto the best. +The class :py:class:`~CompStats.Difference` has the :py:class:`~CompStats.Difference.plot` method that can be used to depict the difference with respectto the best. >>> diff.plot() .. image:: digits_difference.png +Multi-measure Perf +-------------------- + +A single competition can also be evaluated with more than one measure at once (e.g., macro-F1 together with macro-recall) by passing a list of functions to :py:attr:`score_func`/:py:attr:`error_func`. :py:attr:`score_func` and :py:attr:`error_func` can even be combined to mix score-type and error-type measures, with different Bigger-is-Better (BiB) directions, into a single :py:class:`~CompStats.interface.Perf` instance. Every measure is evaluated on the same bootstrap resamples, so comparisons across algorithms remain paired for each measure. + +Every :py:mod:`CompStats.metrics` wrapper exposes a ``.measure`` factory (e.g. :py:func:`~CompStats.metrics.f1_score.measure`) that builds the tagged callable used internally as :py:attr:`score_func`/:py:attr:`error_func`; call it directly to compose several measures, as shown next. + +>>> from CompStats.interface import Perf +>>> from CompStats.metrics import f1_score, recall_score +>>> mperf = Perf(y_val, hy, forest=ens.predict(X_val), +... score_func=[f1_score.measure(average='macro'), +... recall_score.measure(average='macro')], +... measure_names=['macro-F1', 'macro-Recall']) +>>> mperf + +Statistic with its standard error (se) +statistic (se) +0.9783 (0.0061), 0.9786 (0.0060) <= forest +0.9440 (0.0098), 0.9442 (0.0098) <= alg-1 + +:py:attr:`measure_names` is optional; when omitted, each measure is labeled with its function's ``__name__`` (e.g. ``f1_score``, ``recall_score``). The properties :py:func:`~CompStats.interface.Perf.statistic`, :py:func:`~CompStats.interface.Perf.se`, and :py:func:`~CompStats.interface.Perf.ci` return one value per measure for every system. + +>>> mperf.statistic +{'forest': array([0.97828319, 0.97855524]), 'alg-1': array([0.94399193, 0.94424915])} +>>> mperf.se +{'forest': array([0.00605715, 0.00595743]), 'alg-1': array([0.0098302 , 0.00978099])} +>>> mperf.ci +{'alg-1': (array([0.92393337, 0.92473771]), array([0.96193002, 0.96198806])), 'forest': (array([0.96618741, 0.9666813 ]), array([0.98902657, 0.98936809]))} + +:py:func:`~CompStats.interface.Perf.plot` facets the resulting figure by measure, and :py:func:`~CompStats.interface.Perf.difference` reports one p-value per measure for each system compared to the best. + +>>> mperf.plot() +>>> mperf.difference() + +difference p-values +forest, forest <= Best +0.0000, 0.0000 <= alg-1 +1.0000, 1.0000 <= forest + +The convenience wrappers :py:func:`~CompStats.metrics.macro_f1`, :py:func:`~CompStats.metrics.macro_recall`, and :py:func:`~CompStats.metrics.macro_precision` are ready-made, multi-measure-friendly shortcuts for macro-averaged F1, recall, and precision; each also exposes its own ``.measure`` factory (e.g. :py:func:`~CompStats.metrics.macro_f1.measure`), so they can be combined the same way as any other :py:mod:`CompStats.metrics` wrapper. + +>>> from CompStats.metrics import macro_f1, macro_recall +>>> Perf(y_val, hy, forest=ens.predict(X_val), +... score_func=[macro_f1.measure(), macro_recall.measure()]) + +Statistic with its standard error (se) +statistic (se) +0.9783 (0.0061), 0.9786 (0.0060) <= forest +0.9440 (0.0101), 0.9442 (0.0101) <= alg-1 + .. automodule:: CompStats.metrics :members: \ No newline at end of file diff --git a/quarto/CompStats.qmd b/quarto/CompStats.qmd index 2db0cbb..dde9f2a 100644 --- a/quarto/CompStats.qmd +++ b/quarto/CompStats.qmd @@ -110,4 +110,62 @@ hist = HistGradientBoostingClassifier().fit(X_train, y_train) _ = score(hist.predict(X_val), name='Hist. Grad. Boost. Tree') score.plot() ``` +::: + +# Multi-measure Support + +## Column + +Sometimes a competition needs to be evaluated with more than one measure at once, e.g., macro-F1 together with macro-recall. `CompStats` supports this by passing a list of functions to `score_func`/`error_func`; the two can even be combined to mix score-type and error-type measures, with different Bigger-is-Better directions, into a single `Perf` instance. Every measure is evaluated on the same bootstrap resamples, so comparisons across algorithms remain paired for each measure. + +Every `CompStats.metrics` wrapper exposes a `.measure` factory (e.g. `f1_score.measure`) that builds the tagged callable used internally as `score_func`/`error_func`; call it directly to compose several measures, as shown next. + +::: {.card title="Combining two measures" .flow} +```{python} +#| echo: true + +from CompStats.interface import Perf +from CompStats.metrics import f1_score, recall_score + +mperf = Perf(y_val, hy, forest=ens.predict(X_val), + score_func=[f1_score.measure(average='macro'), + recall_score.measure(average='macro')], + measure_names=['macro-F1', 'macro-Recall']) +mperf +``` +::: + +`measure_names` is optional; when omitted, each measure is labeled with its function's `__name__`. The properties `statistic`, `se`, and `ci` return one value per measure for every system. + +::: {.card title="Inspecting the measures" .flow} +```{python} +#| echo: true + +mperf.statistic +``` +::: + +`plot` facets the resulting figure by measure, and `difference` reports one p-value per measure for each system compared to the best. + +::: {.card title="Difference across measures" .flow} +```{python} +#| echo: true + +mperf.difference() +``` +::: + +## Column + +The convenience wrappers `macro_f1`, `macro_recall`, and `macro_precision` are ready-made, multi-measure-friendly shortcuts for macro-averaged F1, recall, and precision; each also exposes its own `.measure` factory, so they can be combined the same way as any other `CompStats.metrics` wrapper. + +::: {.card title="Convenience wrappers" .flow} +```{python} +#| echo: true + +from CompStats.metrics import macro_f1, macro_recall + +Perf(y_val, hy, forest=ens.predict(X_val), + score_func=[macro_f1.measure(), macro_recall.measure()]) +``` ::: \ No newline at end of file