From 5b259b3d6af482ea37750d9041fd403296190218 Mon Sep 17 00:00:00 2001 From: thc1006 <84045975+thc1006@users.noreply.github.com> Date: Sat, 27 Jun 2026 20:44:04 +0800 Subject: [PATCH] test: add golden-master regression for scenario #0 (issue #38) Re-runs scenario #0 (deterministic: static balloons, fixed seed, ideal sensors, no Monte Carlo) with the default agent and compares against a committed baseline, guarding against unintended physics changes when ActiveRocketPy is updated. - Compares popped count (exact, == 10), rocket position trajectory (rtol 3%, atol 1 m), and flight duration (within 2%). - Baseline is downsampled post-launch positions; regenerate deliberately via tests/baselines/regenerate_scenario_0.py and review the diff. - Side-effect free runner; guarded with skipUnless like the other runtime tests. --- tests/baselines/regenerate_scenario_0.py | 46 ++ tests/baselines/scenario_0.json | 613 +++++++++++++++++++++++ tests/test_scenario0_regression.py | 114 +++++ 3 files changed, 773 insertions(+) create mode 100644 tests/baselines/regenerate_scenario_0.py create mode 100644 tests/baselines/scenario_0.json create mode 100644 tests/test_scenario0_regression.py diff --git a/tests/baselines/regenerate_scenario_0.py b/tests/baselines/regenerate_scenario_0.py new file mode 100644 index 0000000..68a6886 --- /dev/null +++ b/tests/baselines/regenerate_scenario_0.py @@ -0,0 +1,46 @@ +"""Regenerate the scenario #0 regression baseline (issue #38). + +Run from the repository root *after a deliberate physics change* (and review the +resulting diff before committing): + + PYTHONPATH=. python tests/baselines/regenerate_scenario_0.py + +The baseline is tied to the current repository + ActiveRocketPy submodule state, +so it must be regenerated whenever the rocket physics is intentionally updated. +""" + +import json +from pathlib import Path + +from tests.test_scenario0_regression import ( + AGENT_KWARGS, + DOWNSAMPLE_STRIDE, + post_launch_positions, + run_scenario_0, +) + +OUTPUT_PATH = Path(__file__).parent / "scenario_0.json" + + +def main(): + positions, popped = run_scenario_0() + baseline = { + "scenario_number": 0, + "random_seed": 0, + "agent": "AttitudeRateControlAgent", + "agent_kwargs": AGENT_KWARGS, + "downsample_stride": DOWNSAMPLE_STRIDE, + "num_steps_full": int(positions.shape[0]), + "popped_count": int(popped), + "rocket_position_downsampled": post_launch_positions(positions).tolist(), + } + with open(OUTPUT_PATH, "w", encoding="utf-8") as output_file: + json.dump(baseline, output_file, indent=2) + print( + f"wrote {OUTPUT_PATH} " + f"(steps={baseline['num_steps_full']}, popped={baseline['popped_count']})" + ) + + +if __name__ == "__main__": + main() diff --git a/tests/baselines/scenario_0.json b/tests/baselines/scenario_0.json new file mode 100644 index 0000000..03741f1 --- /dev/null +++ b/tests/baselines/scenario_0.json @@ -0,0 +1,613 @@ +{ + "scenario_number": 0, + "random_seed": 0, + "agent": "AttitudeRateControlAgent", + "agent_kwargs": { + "rate_targets": [ + 0.0, + 0.0, + 0.0 + ], + "launch_time": 1 + }, + "downsample_stride": 50, + "num_steps_full": 6012, + "popped_count": 10, + "rocket_position_downsampled": [ + [ + -1.6839707610917858e-09, + 3.564997217377111e-16, + 20.002549511995664 + ], + [ + -1.0571302383648144e-05, + 8.412649859742009e-11, + 20.399111796562785 + ], + [ + -6.571275485667554e-05, + 9.876207023531452e-10, + 21.330893837119294 + ], + [ + -0.00020110526701182477, + 4.4080511497538785e-09, + 22.80914612728804 + ], + [ + -0.0004520399955784506, + 1.2970685381652024e-08, + 24.845047987216333 + ], + [ + -0.00085292725086832, + 3.016105566607069e-08, + 27.449698581226436 + ], + [ + -0.0014368367970977833, + 6.022919060189882e-08, + 30.63409516726168 + ], + [ + -0.0022350879102580154, + 1.080698753779587e-07, + 34.40913251898384 + ], + [ + -0.0032769048939612637, + 1.790846924463149e-07, + 38.785591954371505 + ], + [ + -0.004589151191046608, + 2.7903272635906306e-07, + 43.77412568878896 + ], + [ + -0.006196154174906679, + 4.1387762535104413e-07, + 49.38524884177982 + ], + [ + -0.008119629494062275, + 5.89639140161842e-07, + 55.62932855712281 + ], + [ + -0.01037871004093352, + 8.122572276954227e-07, + 62.51657162814403 + ], + [ + -0.012990081074187086, + 1.0874762876657448e-06, + 70.0570160014119 + ], + [ + -0.015968218491924924, + 1.4207561122439115e-06, + 78.26051623479741 + ], + [ + -0.019325723180388662, + 1.8172147301344644e-06, + 87.13673573247561 + ], + [ + -0.023073740288045253, + 2.2816065512783757e-06, + 96.6951352423587 + ], + [ + -0.027222448605138783, + 2.818337256727633e-06, + 106.9449613002894 + ], + [ + -0.031781602535760616, + 3.4315148151954615e-06, + 117.89523690862823 + ], + [ + -0.03676110698880097, + 4.125034000229496e-06, + 129.55475014504404 + ], + [ + -0.04217160495015993, + 4.902690022075723e-06, + 141.93204683852431 + ], + [ + -0.048025057025851074, + 5.768315448347836e-06, + 155.03541855479273 + ], + [ + -0.05433529443604968, + 6.7259336683939e-06, + 168.87289639383252 + ], + [ + -0.06111852827440976, + 7.779921653809515e-06, + 183.45224213095807 + ], + [ + -0.06839380096021018, + 8.93517487546531e-06, + 198.78093555032666 + ], + [ + -0.07618337109040887, + 1.0197268021795575e-05, + 214.86617214405265 + ], + [ + -0.08451302406101736, + 1.15726057916632e-05, + 231.71485449985772 + ], + [ + -0.09341230639139539, + 1.3068559731815799e-05, + 249.33358140973317 + ], + [ + -0.10291468709258966, + 1.4693588611668966e-05, + 267.7286495860788 + ], + [ + -0.11305764767790659, + 1.6457340842160717e-05, + 286.90604405804646 + ], + [ + -0.12388270973459946, + 1.83707394746777e-05, + 306.8714325920476 + ], + [ + -0.13543540924219075, + 2.0446051307872837e-05, + 327.6301638720605 + ], + [ + -0.14776522632241712, + 2.269694230343713e-05, + 349.18726348025115 + ], + [ + -0.16092547959331516, + 2.5138522122797565e-05, + 371.547425870094 + ], + [ + -0.17497320012468354, + 2.7787381854279604e-05, + 394.7150220720344 + ], + [ + -0.18996898770030224, + 3.0661626990573416e-05, + 418.69409973057395 + ], + [ + -0.20597685648807132, + 3.378090859770433e-05, + 443.4883693177766 + ], + [ + -0.2230644343475757, + 3.716656969018691e-05, + 469.0996438064178 + ], + [ + -0.24130393714222625, + 4.084204784070951e-05, + 495.5258567120096 + ], + [ + -0.2607717934255657, + 4.4832895685498427e-05, + 522.7641733585793 + ], + [ + -0.2815482536845713, + 4.916679052450254e-05, + 550.8114955137588 + ], + [ + -0.30371714012558654, + 5.3873580685486285e-05, + 579.6644691543897 + ], + [ + -0.3273656465387332, + 5.898534590481246e-05, + 609.3194992380141 + ], + [ + -0.352584176508395, + 6.453647054457166e-05, + 639.7727593421967 + ], + [ + -0.37946621855005563, + 7.056372964572014e-05, + 671.0202066567837 + ], + [ + -0.4081082480258238, + 7.7106385611159e-05, + 703.0575949797576 + ], + [ + -0.4386096495552378, + 8.420629372862776e-05, + 735.8804854913432 + ], + [ + -0.4710726574914859, + 9.190801539978243e-05, + 769.4842593284947 + ], + [ + -0.5056023114333767, + 0.00010025893780618044, + 803.8641330800716 + ], + [ + -0.5423064180586958, + 0.00010930939755859502, + 839.0151692542238 + ], + [ + -0.5812955229035692, + 0.00011911280866408835, + 874.9322895515436 + ], + [ + -0.6226828898427113, + 0.00012972579408270806, + 911.6102888744468 + ], + [ + -0.666584484029597, + 0.00014120831958967147, + 949.0438476635286 + ], + [ + -0.7131189595610133, + 0.00015362382999928873, + 987.2275455236096 + ], + [ + -0.7624076481415989, + 0.0001670393867246597, + 1026.1558727475638 + ], + [ + -0.8145745530473908, + 0.00018152580769861977, + 1065.8232440456136 + ], + [ + -0.8697463444292, + 0.00019715780866541506, + 1106.224010819118 + ], + [ + -0.9280523581569751, + 0.00021401414645709504, + 1147.3524738155388 + ], + [ + -0.9896245964639214, + 0.0002321777638529419, + 1189.202894746165 + ], + [ + -1.0545977326742264, + 0.0002517359367247873, + 1231.7695086590002 + ], + [ + -1.1231082228551255, + 0.0002727801609044962, + 1275.0459074368555 + ], + [ + -1.1934115876679707, + 0.00029497819048940945, + 1317.131045854382 + ], + [ + -1.2662638482990738, + 0.0003191381957369389, + 1356.3086493918556 + ], + [ + -1.3403966149806632, + 0.00034482847733953296, + 1392.6432717786915 + ], + [ + -1.4158528217602075, + 0.0003721537694500129, + 1426.1931036237427 + ], + [ + -1.4928184305413081, + 0.0004013060060074859, + 1457.0104843752435 + ], + [ + -1.5707132530477579, + 0.0004321197112658861, + 1485.1423471014773 + ], + [ + -1.6495172215817493, + 0.00046466739281225705, + 1510.6305996858584 + ], + [ + -1.729183348090482, + 0.0004990124626353674, + 1533.5124592873597 + ], + [ + -1.8094490352643429, + 0.0005351146765100272, + 1553.8207408187325 + ], + [ + -1.8900757802530628, + 0.0005729334847761912, + 1571.5841038492151 + ], + [ + -1.9708749404841994, + 0.000612441844369116, + 1586.8272640037821 + ], + [ + -2.051630983489959, + 0.0006535913799438475, + 1599.5711721777382 + ], + [ + -2.132073023758924, + 0.0006962973886389526, + 1609.8331638882241 + ], + [ + -2.211892369011248, + 0.0007404468925188164, + 1617.627080987497 + ], + [ + -2.29079739465414, + 0.0007859314926911473, + 1622.96336741302 + ], + [ + -2.3686540325283483, + 0.0008327381935873848, + 1625.8491394053585 + ], + [ + -2.4457894673184923, + 0.0008811499497958456, + 1626.2882288021324 + ], + [ + -2.5181476085762498, + 0.0009286623012529134, + 1624.281292329971 + ], + [ + -2.5550803526714803, + 0.0009556029747260294, + 1619.8264542745146 + ], + [ + -2.489093651505675, + 0.0009182497399375386, + 1612.9220965814302 + ], + [ + -2.2159164874046984, + 0.0007484533059931559, + 1603.5795500826391 + ], + [ + -1.6041758363572673, + 0.00035869240212103723, + 1591.8605420262627 + ], + [ + -0.5518713945705036, + -0.00032330026048903334, + 1577.944571730845 + ], + [ + 0.8432677636727125, + -0.0012470836842200573, + 1562.1502994737884 + ], + [ + 2.0673227340476705, + -0.002098871564034423, + 1544.6876784379358 + ], + [ + 3.056764032152677, + -0.002829371669910768, + 1525.261838753259 + ], + [ + 4.76437025843169, + -0.004029868754606541, + 1504.3300834967743 + ], + [ + 6.907196775664617, + -0.005537579480287317, + 1482.3398752146466 + ], + [ + 9.819785910186631, + -0.007564946157858897, + 1459.2354198254548 + ], + [ + 12.548628429784724, + -0.009538704752050485, + 1434.325133064808 + ], + [ + 13.514056046112104, + -0.010482200811298532, + 1408.0020988697504 + ], + [ + 13.656831185138792, + -0.01095967560698507, + 1379.9416926084584 + ], + [ + 15.554982442763597, + -0.01250183626307562, + 1350.2335809335398 + ], + [ + 17.21883180575756, + -0.01391925817322066, + 1318.5288334443014 + ], + [ + 18.043154491355793, + -0.014884898087787111, + 1284.7896978466488 + ], + [ + 19.61199959050112, + -0.016320081644152645, + 1249.0408909628748 + ], + [ + 20.85957304523814, + -0.017581315054793618, + 1211.246143147733 + ], + [ + 22.098845585881453, + -0.018879876428591007, + 1171.445606731479 + ], + [ + 23.431889805706493, + -0.02025003403972233, + 1129.6816698116584 + ], + [ + 24.64840408687379, + -0.021583752747288572, + 1085.9916305766783 + ], + [ + 25.928123049603297, + -0.02297588812863238, + 1040.4287801540188 + ], + [ + 27.14969526462085, + -0.02435979180953873, + 993.0398734252464 + ], + [ + 28.383407906711792, + -0.025771870160842408, + 943.8785960910492 + ], + [ + 29.5925943788057, + -0.027192235771008075, + 892.9968890417614 + ], + [ + 30.791596789637875, + -0.028626109804347096, + 840.4488839844379 + ], + [ + 31.97713010237086, + -0.03007179456543693, + 786.2889257760954 + ], + [ + 33.14672448145517, + -0.03152595585026869, + 730.5719044066002 + ], + [ + 34.30291962768768, + -0.03298916043558614, + 673.3531655610402 + ], + [ + 35.44370225238458, + -0.034459110591997445, + 614.6881685236697 + ], + [ + 36.569257903058784, + -0.03593466967618154, + 554.6324849546223 + ], + [ + 37.6794872365691, + -0.03741468777063867, + 493.24160923746075 + ], + [ + 38.77401088482113, + -0.03889781003192398, + 430.5708276447076 + ], + [ + 39.85267629228074, + -0.04038281805948896, + 366.67509987696894 + ], + [ + 40.91534714543381, + -0.04186853354824656, + 301.6089326247653 + ], + [ + 41.96188013087519, + -0.04335379036318722, + 235.4262624064347 + ], + [ + 42.992164921498514, + -0.04483745990599321, + 168.18034525711118 + ], + [ + 44.006120680502896, + -0.04631845420106061, + 99.92365180120734 + ], + [ + 45.00368950463338, + -0.04779572378281738, + 30.70776868398184 + ] + ] +} \ No newline at end of file diff --git a/tests/test_scenario0_regression.py b/tests/test_scenario0_regression.py new file mode 100644 index 0000000..c661f49 --- /dev/null +++ b/tests/test_scenario0_regression.py @@ -0,0 +1,114 @@ +"""Golden-master regression test for scenario #0 (issue #38). + +Scenario #0 is deterministic -- static balloons, a fixed seed, ideal sensors and +no Monte Carlo -- so a fixed agent produces a reproducible rocket trajectory. +This test re-runs it and compares against a committed baseline, guarding against +unintended changes to the physics (for example when ActiveRocketPy is updated). + +The baseline is tied to the current (repository, ActiveRocketPy submodule) state. +When a physics change is *intended*, regenerate it deliberately with +``tests/baselines/regenerate_scenario_0.py`` and review the diff. + +Scope (v1): the popped-balloon count (exact) and the rocket position trajectory +(~3% tolerance). Attitude and velocity are intentionally left for a later +iteration to keep this guard small and its tolerances simple. +""" + +import json +import unittest +from pathlib import Path + +try: + import numpy as np + + from BalloonPoppingGymEnv.agents.example_agents import AttitudeRateControlAgent + from BalloonPoppingGymEnv.envs.balloon_world import BalloonPoppingEnv + from BalloonPoppingGymEnv.evaluation.evaluate import load_scenario_parameters + + _STACK_AVAILABLE = True +except ImportError: + _STACK_AVAILABLE = False + +BASELINE_PATH = Path(__file__).parent / "baselines" / "scenario_0.json" + +# Fixed agent configuration the baseline was generated with. +AGENT_KWARGS = {"rate_targets": [0.0, 0.0, 0.0], "launch_time": 1} +DOWNSAMPLE_STRIDE = 50 + +# Same-machine runs are bit-identical; these tolerances absorb cross-platform +# float drift only (~3% per issue #38). atol is a metre floor for coordinates +# near zero. +POSITION_RTOL = 0.03 +POSITION_ATOL = 1.0 +# A gross change in flight duration is a regression; a one-step cross-platform +# difference is not. +STEP_COUNT_REL_TOL = 0.02 + + +def run_scenario_0(): + """Run scenario #0 with the fixed agent and seed. + + Returns ``(positions, popped)`` where ``positions`` is the per-step rocket + centre-of-mass position ``(num_steps, 3)`` -- including the NaN rows before + launch -- and ``popped`` is the final cumulative popped-balloon count. Side + effect free: it does not write trajectory files. + """ + scenario_params, given_params = load_scenario_parameters(0) + env = BalloonPoppingEnv(render_mode=None, parameters=scenario_params) + agent = AttitudeRateControlAgent(given_params, **AGENT_KWARGS) + observation, _ = env.reset(seed=scenario_params["scenario"]["random_seed"]) + terminated = False + while not terminated: + action = agent.get_action(observation) + observation, _, terminated, _, _ = env.step(action) + # env.trajectories holds a per-step copy of the true rocket state. + rocket_states = np.array( + [step["rocket_states"] for step in env.trajectories], dtype=float + ) + return rocket_states[:, :3], int(env._popped_count) + + +def post_launch_positions(positions): + """Drop the pre-launch NaN rows and downsample to a clean ``(M, 3)`` array.""" + launched = ~np.isnan(positions).any(axis=1) + return positions[launched][::DOWNSAMPLE_STRIDE] + + +@unittest.skipUnless(_STACK_AVAILABLE, "simulation stack not installed") +class TestScenario0Regression(unittest.TestCase): + """Re-run scenario #0 once and compare it against the committed baseline.""" + + @classmethod + def setUpClass(cls): + with open(BASELINE_PATH, encoding="utf-8") as baseline_file: + cls.baseline = json.load(baseline_file) + positions, cls.popped = run_scenario_0() + cls.num_steps = positions.shape[0] + cls.positions = post_launch_positions(positions) + + def test_popped_count_matches_baseline(self): + self.assertEqual(self.popped, self.baseline["popped_count"]) + + def test_flight_duration_matches_baseline(self): + expected = self.baseline["num_steps_full"] + drift = abs(self.num_steps - expected) / expected + self.assertLessEqual( + drift, + STEP_COUNT_REL_TOL, + f"flight step count {self.num_steps} drifted from baseline {expected}", + ) + + def test_rocket_position_trajectory_matches_baseline(self): + expected = np.array(self.baseline["rocket_position_downsampled"], dtype=float) + actual = self.positions + overlap = min(len(expected), len(actual)) + np.testing.assert_allclose( + actual[:overlap], + expected[:overlap], + rtol=POSITION_RTOL, + atol=POSITION_ATOL, + ) + + +if __name__ == "__main__": + unittest.main()