From 4dfe8d2d19ff72c0f73050298158f53d727cefb3 Mon Sep 17 00:00:00 2001 From: Haruki Nishimura Date: Thu, 2 Jul 2026 18:38:26 -0700 Subject: [PATCH 1/3] Migrate base interfaces to statistical_comparison_core and statistical_comparison_helpers - Base hypothesis testing enums/classes are imported from the new statistical_comparison_core instead of sequentialized_barnard_tests - Remove redundante plotting and CLD code from this package; they are migrated to statistical_comparison_helpers - Add statistical-comparison-core and statistical-comparison-helpers as new dependencies. Also add cvxpy. - Update setup.py - Update tools/plotting.py deprecation shims --- .gitignore | 8 + nscore/batch.py | 3 +- nscore/nonparametric_nsm.py | 2 +- nscore/nsm.py | 2 +- nscore/savi.py | 2 +- nscore/tools/plotting.py | 226 ++++++------------------- nscore/utils/utils_wsr.py | 2 +- nscore/wsr.py | 4 +- setup.py | 42 ++--- tests/methods/test_binary_nsm.py | 2 +- tests/methods/test_continuous_nsm.py | 2 +- tests/test_nscore_deprecation_shims.py | 88 ++++++++++ 12 files changed, 174 insertions(+), 209 deletions(-) create mode 100644 .gitignore create mode 100644 tests/test_nscore_deprecation_shims.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6d0c25b --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +__pycache__/ +*.py[cod] +*.egg-info/ +*.egg +dist/ +build/ +.pytest_cache/ +.eggs/ diff --git a/nscore/batch.py b/nscore/batch.py index 72f47bc..6c5fcd2 100644 --- a/nscore/batch.py +++ b/nscore/batch.py @@ -6,9 +6,8 @@ import numpy as np from numpy.typing import ArrayLike from scipy.stats import barnard_exact -import sequentialized_barnard_tests as sbt -from sequentialized_barnard_tests.base import ( +from statistical_comparison_core import ( Decision, Hypothesis, MirroredTestMixin, diff --git a/nscore/nonparametric_nsm.py b/nscore/nonparametric_nsm.py index a32dd37..f361005 100644 --- a/nscore/nonparametric_nsm.py +++ b/nscore/nonparametric_nsm.py @@ -8,7 +8,7 @@ import numpy as np from scipy.stats import beta, dirichlet -from sequentialized_barnard_tests.base import ( +from statistical_comparison_core import ( Decision, Hypothesis, MirroredTestMixin, diff --git a/nscore/nsm.py b/nscore/nsm.py index a520234..6f7293c 100644 --- a/nscore/nsm.py +++ b/nscore/nsm.py @@ -8,7 +8,7 @@ import numpy as np from scipy.stats import beta, dirichlet -from sequentialized_barnard_tests.base import ( +from statistical_comparison_core import ( Decision, Hypothesis, MirroredTestMixin, diff --git a/nscore/savi.py b/nscore/savi.py index b2c2a48..f416916 100644 --- a/nscore/savi.py +++ b/nscore/savi.py @@ -14,7 +14,7 @@ from numpy.typing import ArrayLike from scipy.stats import bernoulli, beta, dirichlet, multinomial -from sequentialized_barnard_tests.base import ( +from statistical_comparison_core import ( Decision, Hypothesis, MirroredTestMixin, diff --git a/nscore/tools/plotting.py b/nscore/tools/plotting.py index 237f886..d478e78 100644 --- a/nscore/tools/plotting.py +++ b/nscore/tools/plotting.py @@ -1,100 +1,35 @@ from typing import Dict, List, Optional, Tuple, Union +import warnings + from matplotlib.cm import get_cmap import matplotlib.pyplot as plt import numpy as np from scipy import stats -from sequentialized_barnard_tests import Decision, Hypothesis +from statistical_comparison_core import Decision, Hypothesis from nscore.nonparametric_nsm import MirroredContinuousNsmTest +import statistical_comparison_helpers as _sch + def compact_letter_display( significant_pair_list: List[Tuple[str, str]], sorted_model_list: List[str], ) -> List[str]: """Generates Compact Letter Display (CLD) given a list of significant - pairs and a list of models. CLD is Based on "An Algorithm for a - Letter-Based Representation of All-Pairwise Comparisons" by Piepho - (2004). + pairs and a list of models. - Args: - significant_pair_list: A list containing tuples of model names that - were deemed significantly different by each A/B test. - sorted_model_list: A list of model names sorted by performance in - descending order. - - Returns: - A list of letters representing CLD for the corresponding models. + .. deprecated:: + Use ``statistical_comparison_helpers.compact_letter_display`` instead. """ - num_models = len(sorted_model_list) - - # Map model names to indices. - model_to_index = {model: idx for idx, model in enumerate(sorted_model_list)} - # Convert significant pairs from names to indices. - significant_index_pairs = [ - (model_to_index[m1], model_to_index[m2]) for m1, m2 in significant_pair_list - ] - - # --- Inner helper to remove redundant columns --- - def remove_redundant_columns(matrix): - changed = True - while changed: - changed = False - for i in range(len(matrix)): - for j in range(len(matrix)): - if i != j: - indices_i = {idx for idx, char in enumerate(matrix[i]) if char} - indices_j = {idx for idx, char in enumerate(matrix[j]) if char} - if indices_i.issubset(indices_j): - matrix.pop(i) - changed = True - break - if changed: - break - return matrix - - # --- Main algorithm --- - # Start with a single column of 'a's for all models. - letter_matrix = [["a"] * num_models] - - # For each significant pair, update the letter matrix. - for model_idx1, model_idx2 in significant_index_pairs: - while any(col[model_idx1] and col[model_idx2] for col in letter_matrix): - for col_index, letter_column in enumerate(letter_matrix): - if letter_column[model_idx1] and letter_column[model_idx2]: - new_column = letter_column.copy() - new_column[model_idx1] = "" - letter_column[model_idx2] = "" - letter_matrix[col_index] = letter_column - letter_matrix.append(new_column) - letter_matrix = remove_redundant_columns(letter_matrix) - break # re-check with the while condition - - # --- Reassign letters based on sorted columns --- - def first_nonempty_position(column): - for pos, char in enumerate(column): - if char: - return pos - return len(column) - - letter_matrix.sort(key=first_nonempty_position) - - for idx, column in enumerate(letter_matrix): - replacement_letter = chr(ord("a") + idx) - letter_matrix[idx] = [replacement_letter if char else "" for char in column] - - # --- Build final CLD output for each model --- - final_display = [] - for model_idx in range(num_models): - letters = "".join( - letter_matrix[col_idx][model_idx] - for col_idx in range(len(letter_matrix)) - if letter_matrix[col_idx][model_idx] - ) - final_display.append(letters) - - return final_display + warnings.warn( + "nscore.tools.plotting.compact_letter_display is deprecated. " + "Use statistical_comparison_helpers.compact_letter_display instead.", + DeprecationWarning, + stacklevel=2, + ) + return _sch.compact_letter_display(significant_pair_list, sorted_model_list) def compare_success_and_get_cld( @@ -192,7 +127,7 @@ def compare_success_and_get_cld( reverse=True, ) ] - letters_list = compact_letter_display( + letters_list = _sch.compact_letter_display( input_list_to_cld, models_sorted_by_success_rates ) if verbose: @@ -248,25 +183,20 @@ def draw_samples_from_beta_posterior( beta_prior: float = 1, ) -> np.ndarray: """Draw samples from the beta posterior distribution given a success array. - These samples can be used to estimate the posterior distribution of the - success rate of a Bernoulli process. Note that the default prior parameters - of (1, 1) correspond to a uniform prior. - - Args: - success_array: A binary array with True/False indicating success/failure. - rng: A numpy random Generator instance. - num_samples: Optional number of samples to draw. Defaults to 10000. - alpha_prior: Optional alpha parameter of the beta prior. Defaults to 1. - beta_prior: Optional beta parameter of the beta prior. Defaults to 1. - Returns: - Samples drawn from the beta posterior distribution. + .. deprecated:: + Use ``statistical_comparison_helpers.draw_samples_from_beta_posterior`` instead. """ - n_trials = len(success_array) - n_successes = np.sum(success_array) - n_failures = n_trials - n_successes - posterior = stats.beta(alpha_prior + n_successes, beta_prior + n_failures) - return posterior.rvs(num_samples, random_state=rng) + warnings.warn( + "nscore.tools.plotting.draw_samples_from_beta_posterior is deprecated. " + "Use statistical_comparison_helpers.draw_samples_from_beta_posterior instead.", + DeprecationWarning, + stacklevel=2, + ) + return _sch.draw_samples_from_beta_posterior( + success_array, rng, num_samples=num_samples, + alpha_prior=alpha_prior, beta_prior=beta_prior, + ) def plot_model_comparison( @@ -281,81 +211,31 @@ def plot_model_comparison( height: int = 4, dpi: int = 100, ) -> Union[None, plt.Figure]: - """Makes a violin plot of success rate estimates with corresponding CLD letters - for policy comparison. + """Makes a violin plot of success rate estimates with corresponding CLD letters. - Args: - model_name_list: A list of model names. - success_arrays: A list of arrays indicating success/failure for each model. - cld_letters: A list of CLD letters corresponding to each model. - rng: A numpy random Generator instance for posterior sampling. - output_path: Optional file path to save the plot. If None, the plot will not - be saved but returned as a matplotlib Figure object. Defaults to None. - title: Optional title for the plot. Defaults to None. - add_legend: Whether to show legend on the plot. Defaults to False. - unit_width: Figure width per model. Defaults to 6. - height: Figure height. Defaults to 4. - dpi: Resolution of the saved plot. Defaults to 100. - - Returns: - If output_path is None, returns a matplotlib Figure object containing - the plot. Otherwise, saves the plot to the specified path and returns None. + .. deprecated:: + Use ``statistical_comparison_helpers.plot_model_comparison`` instead. """ - num_models = len(model_name_list) - - posterior_samples = [] - means = [] - - for success_array in success_arrays: - samples = draw_samples_from_beta_posterior(success_array, rng) - posterior_samples.append(samples) - means.append(np.mean(samples)) - - fig, ax = plt.subplots(figsize=(max(unit_width, num_models), height), dpi=dpi) - - cmap = get_cmap("tab10") - colors = [cmap(i % 10) for i in range(num_models)] - - parts = ax.violinplot( - posterior_samples, - positions=np.arange(num_models), - showmeans=True, - showmedians=False, - showextrema=False, - widths=0.8, + warnings.warn( + "nscore.tools.plotting.plot_model_comparison is deprecated. " + "Use statistical_comparison_helpers.plot_model_comparison instead.", + DeprecationWarning, + stacklevel=2, ) - for pc, color in zip(parts["bodies"], colors): - pc.set_facecolor(color) - pc.set_alpha(0.6) - parts["cmeans"].set_color("black") - parts["cmeans"].set_linewidth(0.8) - - # Add CLD labels - for i, (x, y, label) in enumerate(zip(np.arange(num_models), means, cld_letters)): - ax.text( - x + 0.15, - y + 0.03, - label, - fontsize=12, - fontweight="bold", - color="black", - verticalalignment="center", - zorder=4, - ) - - ax.set_xticks(np.arange(num_models)) - ax.set_xticklabels(model_name_list, rotation=0, ha="center") - ax.set_ylim(0.0, 1.0) - ax.set_ylabel("Success Rate") - if title is not None: - ax.set_title(title) - if add_legend: - ax.legend(parts["bodies"], model_name_list, loc="best") - plt.tight_layout() - - if output_path is not None: - plt.savefig(output_path, dpi=300) - plt.close() - print(f"Saved a PNG plot to {output_path}") - else: - return fig \ No newline at end of file + from statistical_comparison_helpers.plotting import plot_model_comparison as _plot + from statistical_comparison_helpers.posterior import Binary + + return _plot( + model_name_list, + success_arrays, + cld_letters, + rng, + score=Binary(), + plot_mode="posterior", + output_path=output_path, + title=title, + add_legend=add_legend, + unit_width=unit_width, + height=height, + dpi=dpi, + ) \ No newline at end of file diff --git a/nscore/utils/utils_wsr.py b/nscore/utils/utils_wsr.py index 90124d0..00dccd7 100644 --- a/nscore/utils/utils_wsr.py +++ b/nscore/utils/utils_wsr.py @@ -1,5 +1,5 @@ """ -Docstring for sequentialized_barnard_tests.utils.utils_wsr +Docstring for nscore.utils.utils_wsr """ import copy diff --git a/nscore/wsr.py b/nscore/wsr.py index 412649b..f4807ab 100644 --- a/nscore/wsr.py +++ b/nscore/wsr.py @@ -1,10 +1,10 @@ """ -Docstring for sequentialized_barnard_tests.wsr +Docstring for nscore.wsr """ import numpy as np from nscore.utils.utils_wsr import mean_cs_eff_corrected_membership_accelerated -from sequentialized_barnard_tests.base import Decision, Hypothesis, SequentialTestBase, TestResult +from statistical_comparison_core import Decision, Hypothesis, SequentialTestBase, TestResult from typing import Union diff --git a/setup.py b/setup.py index ce784bf..432b83b 100644 --- a/setup.py +++ b/setup.py @@ -1,28 +1,18 @@ from setuptools import find_packages, setup -# setup( -# name="nscore", -# version="0.0.1", -# description="Sequential statistical hypothesis testing for generalized performance measures.", -# authors=["David Snyder", "Haruki Nishimura"], -# author_emails=["dasnyder@princeton.edu", "haruki.nishimura@tri.global"], -# license="MIT", -# packages=find_packages(), -# package_data={ -# "sequentialized_barnard_tests": [ -# "scripts/synthesize_general_step_policy.py", -# "data/lai_calibration_data.npy", -# "policies/n_max_100_alpha_0.05_shape_parameter_0.0_pnorm_False/policy_compressed.pkl", -# "policies/n_max_200_alpha_0.05_shape_parameter_0.0_pnorm_False/policy_compressed.pkl", -# "policies/n_max_500_alpha_0.05_shape_parameter_0.0_pnorm_False/policy_compressed.pkl", -# ], -# }, -# install_requires=[ -# "binomial_cis", -# "matplotlib", -# "numpy>=1.20", -# "pytest", -# "scipy", -# "tqdm", -# ], -# ) +setup( + name="nscore", + version="0.0.1", + description="Sequential statistical hypothesis testing for generalized performance measures.", + authors=["David Snyder", "Haruki Nishimura"], + author_emails=["dasnyder@princeton.edu", "haruki.nishimura@tri.global"], + packages=find_packages(), + install_requires=[ + "matplotlib", + "numpy>=1.20", + "scipy", + "cvxpy", + "statistical-comparison-core", + "statistical-comparison-helpers", + ], +) \ No newline at end of file diff --git a/tests/methods/test_binary_nsm.py b/tests/methods/test_binary_nsm.py index 48ef9ff..b61890b 100644 --- a/tests/methods/test_binary_nsm.py +++ b/tests/methods/test_binary_nsm.py @@ -7,7 +7,7 @@ import numpy as np import pytest -from sequentialized_barnard_tests.base import Decision, Hypothesis +from statistical_comparison_core import Decision, Hypothesis from nscore.nsm import BernoulliNsmTest paper_data_path = str( diff --git a/tests/methods/test_continuous_nsm.py b/tests/methods/test_continuous_nsm.py index d0eee5d..d219a49 100644 --- a/tests/methods/test_continuous_nsm.py +++ b/tests/methods/test_continuous_nsm.py @@ -7,7 +7,7 @@ import numpy as np import pytest -from sequentialized_barnard_tests.base import Decision, Hypothesis +from statistical_comparison_core import Decision, Hypothesis from nscore.nonparametric_nsm import ContinuousNsmTest paper_data_path = str( diff --git a/tests/test_nscore_deprecation_shims.py b/tests/test_nscore_deprecation_shims.py new file mode 100644 index 0000000..9cff4d5 --- /dev/null +++ b/tests/test_nscore_deprecation_shims.py @@ -0,0 +1,88 @@ +"""Tests for deprecation shims in nscore.tools.plotting.""" + +import warnings + +import numpy as np +import pytest + +import statistical_comparison_helpers as sch + + +class TestCLDShim: + def test_emits_deprecation_warning(self): + from nscore.tools.plotting import compact_letter_display + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + result = compact_letter_display( + [("A", "B")], ["A", "B", "C"] + ) + assert len(w) == 1 + assert issubclass(w[0].category, DeprecationWarning) + assert "deprecated" in str(w[0].message).lower() + + def test_output_matches_statistical_comparison_helpers(self): + from nscore.tools.plotting import compact_letter_display + + pairs = [("A", "B"), ("B", "C")] + models = ["A", "B", "C", "D"] + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + shim_result = compact_letter_display(pairs, models) + + expected = sch.compact_letter_display(pairs, models) + assert shim_result == expected + + def test_returns_list_of_str(self): + from nscore.tools.plotting import compact_letter_display + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + result = compact_letter_display([], ["A", "B"]) + + assert isinstance(result, list) + assert all(isinstance(x, str) for x in result) + + +class TestBetaPosteriorShim: + def test_emits_deprecation_warning(self): + from nscore.tools.plotting import draw_samples_from_beta_posterior + + rng = np.random.default_rng(42) + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + draw_samples_from_beta_posterior(np.array([1, 0, 1]), rng, num_samples=100) + assert len(w) == 1 + assert issubclass(w[0].category, DeprecationWarning) + + def test_output_shape(self): + from nscore.tools.plotting import draw_samples_from_beta_posterior + + rng = np.random.default_rng(42) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + result = draw_samples_from_beta_posterior(np.array([1, 0, 1, 1]), rng, num_samples=500) + assert result.shape == (500,) + + +class TestPlotShim: + def test_emits_deprecation_warning(self): + from nscore.tools.plotting import plot_model_comparison + import matplotlib + matplotlib.use("Agg") + + rng = np.random.default_rng(42) + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + fig = plot_model_comparison( + ["A", "B"], + [np.array([1, 0, 1]), np.array([0, 1, 0])], + ["a", "b"], + rng, + ) + assert len(w) == 1 + assert issubclass(w[0].category, DeprecationWarning) + + import matplotlib.pyplot as plt + plt.close(fig) From 582c65c98c0ba5a44fcb301050d0a4aa9b584c87 Mon Sep 17 00:00:00 2001 From: HarukiNishimura-TRI <131016514+HarukiNishimura-TRI@users.noreply.github.com> Date: Thu, 2 Jul 2026 18:46:07 -0700 Subject: [PATCH 2/3] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- nscore/tools/plotting.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nscore/tools/plotting.py b/nscore/tools/plotting.py index d478e78..4d7e2b4 100644 --- a/nscore/tools/plotting.py +++ b/nscore/tools/plotting.py @@ -2,10 +2,8 @@ import warnings -from matplotlib.cm import get_cmap import matplotlib.pyplot as plt import numpy as np -from scipy import stats from statistical_comparison_core import Decision, Hypothesis from nscore.nonparametric_nsm import MirroredContinuousNsmTest From b0c8f8d0397962fde0e02f14dbef5c48f951f8df Mon Sep 17 00:00:00 2001 From: Haruki Nishimura Date: Thu, 2 Jul 2026 18:53:47 -0700 Subject: [PATCH 3/3] Potential fix for pull request finding --- setup.py | 4 ++-- tests/test_nscore_deprecation_shims.py | 17 +++++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/setup.py b/setup.py index 432b83b..4452cef 100644 --- a/setup.py +++ b/setup.py @@ -4,8 +4,8 @@ name="nscore", version="0.0.1", description="Sequential statistical hypothesis testing for generalized performance measures.", - authors=["David Snyder", "Haruki Nishimura"], - author_emails=["dasnyder@princeton.edu", "haruki.nishimura@tri.global"], + author="David Snyder, Haruki Nishimura", + author_email="dsnyder5@engineering.upenn.edu, haruki.nishimura@tri.global", packages=find_packages(), install_requires=[ "matplotlib", diff --git a/tests/test_nscore_deprecation_shims.py b/tests/test_nscore_deprecation_shims.py index 9cff4d5..21620e8 100644 --- a/tests/test_nscore_deprecation_shims.py +++ b/tests/test_nscore_deprecation_shims.py @@ -17,10 +17,9 @@ def test_emits_deprecation_warning(self): result = compact_letter_display( [("A", "B")], ["A", "B", "C"] ) - assert len(w) == 1 - assert issubclass(w[0].category, DeprecationWarning) - assert "deprecated" in str(w[0].message).lower() - + depr_warnings = [x for x in w if issubclass(x.category, DeprecationWarning)] + assert depr_warnings + assert any("deprecated" in str(x.message).lower() for x in depr_warnings) def test_output_matches_statistical_comparison_helpers(self): from nscore.tools.plotting import compact_letter_display @@ -53,8 +52,9 @@ def test_emits_deprecation_warning(self): with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") draw_samples_from_beta_posterior(np.array([1, 0, 1]), rng, num_samples=100) - assert len(w) == 1 - assert issubclass(w[0].category, DeprecationWarning) + depr_warnings = [x for x in w if issubclass(x.category, DeprecationWarning)] + assert depr_warnings + assert any("deprecated" in str(x.message).lower() for x in depr_warnings) def test_output_shape(self): from nscore.tools.plotting import draw_samples_from_beta_posterior @@ -81,8 +81,9 @@ def test_emits_deprecation_warning(self): ["a", "b"], rng, ) - assert len(w) == 1 - assert issubclass(w[0].category, DeprecationWarning) + depr_warnings = [x for x in w if issubclass(x.category, DeprecationWarning)] + assert depr_warnings + assert any("deprecated" in str(x.message).lower() for x in depr_warnings) import matplotlib.pyplot as plt plt.close(fig)