From 63b66e9d0ab306d604130656f7c0577ffaa65753 Mon Sep 17 00:00:00 2001 From: "D. V. Wiebe" Date: Wed, 8 Jul 2026 10:56:41 -0700 Subject: [PATCH 01/11] test: recover test_call_coco Also: * fix rest_server text fixture to read bodies from GET requests * --json and --yaml turn on quiet mode by default --- coco/client.py | 55 ++++++++++++++++++-------- old_tests/test_call_coco.py | 79 ------------------------------------- tests/coco_runner.py | 36 +++++++++++++++-- tests/rest_server.py | 34 +++++++++------- tests/test_call_coco.py | 52 ++++++++++++++++++++++++ 5 files changed, 142 insertions(+), 114 deletions(-) delete mode 100644 old_tests/test_call_coco.py create mode 100644 tests/test_call_coco.py diff --git a/coco/client.py b/coco/client.py index 7f2364f1..6a448327 100644 --- a/coco/client.py +++ b/coco/client.py @@ -337,7 +337,10 @@ def client_send_request( return True, result silent = client_options["silent"] - refresh_time = client_options["refresh_time"] + if silent: + refresh_time = 0 + else: + refresh_time = client_options["refresh_time"] async def print_queue_size(metric_request_count): try: @@ -383,7 +386,7 @@ async def request_and_wait(): Done task for request result. """ main_request = asyncio.create_task(send_request()) - if not silent: + if refresh_time > 0: metric_request_count = 0 while True: metric_request_count = metric_request_count + 1 @@ -866,16 +869,22 @@ def _bullet(formatter, text): "--json/--yaml", "json", is_flag=True, - default=False, - help="Print style for output. The default is --yaml.", + default=None, + help="Print style for output. If one of these flags is used, --quiet is also " + "turned on, unless --interactive is explicitly used. The default is " + "equivalent to: --interactive --yaml.", ) @click.option( "-q", - "--silent", + "--silent/--interactive", "--quiet", is_flag=True, - default=False, - help="Turn off all output except for the result.", + default=None, + help="Quiet mode ensures the output from coco is decodable as either YAML or " + "JSON by suppressing all output except the result. Quiet mode is " + "automatically turned on if one of the output style flags (--json or --yaml) " + "is used. Use --interactive along with those flags to select an output " + "style without turning on quiet mode.", ) @click.option( "-r", @@ -901,7 +910,8 @@ def _bullet(formatter, text): type=int, default=2, show_default=True, - help="Set refresh time for queue updates to SEC seconds.", + help="Set refresh time for queue updates to SEC seconds. A refresh time of " + "zero disables queue updates completley.", ) @click.option( "--help-config", @@ -917,15 +927,26 @@ def entry( ): """This is the coco client.""" - # legacy --style overrides --json or --yaml - if style: - if style == "json": - json = True - elif style == "yaml": - json = False - else: - # Otherwise, just ignore the option. - pass + # legacy --style overrides --json or --yaml, and doesn't turn on quiet mode + # like those flags do. A --style value which isn't "json" or "yaml" is silently + # ignored. + if style == "json": + json = True + if silent is None: + silent = False + elif style == "yaml": + json = False + if silent is None: + silent = False + elif json is not None: + # --json and --yaml turn on --silent, unless overriden + if silent is None: + silent = True + else: + # Default is --yaml --interactive + json = False + if silent is None: + silent = False # Add all the global options to the click context. obj["options"] = { diff --git a/old_tests/test_call_coco.py b/old_tests/test_call_coco.py deleted file mode 100644 index 77a5f593..00000000 --- a/old_tests/test_call_coco.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Test call forwarding to other coco endpoints.""" - -import pytest -from coco.test import coco_runner, endpoint_farm - -ENDPT_NAME = "proxy" -ENDPT_NAME2 = "end" -ENDPT_NAME3 = "end2" -CONFIG = {"log_level": "INFO"} -ENDPOINTS = { - ENDPT_NAME: { - "call": {"coco": [ENDPT_NAME2, ENDPT_NAME3]}, - "group": "test", - "values": {"foo": "int", "bar": "str"}, - }, - ENDPT_NAME2: {"group": "test", "values": {"foo": "int", "bar": "str"}}, - ENDPT_NAME3: {"group": "test", "values": {"foo": "int", "bar": "str"}}, -} -N_CALLS = 2 - - -def callback(data): - """Reply with the incoming json request.""" - return data - - -N_HOSTS = 2 -CALLBACKS = {ENDPT_NAME2: callback, ENDPT_NAME3: callback} - - -@pytest.fixture -def farm(): - """Create a coco runner.""" - return endpoint_farm.Farm(N_HOSTS, CALLBACKS) - - -@pytest.fixture -def runner(farm): - """Create an endpoint test farm.""" - CONFIG["groups"] = {"test": farm.hosts} - with coco_runner.Runner(CONFIG, ENDPOINTS) as runner: - yield runner - - -def test_forward(farm, runner): - """Test if a request gets forwarded to another coco endpoint.""" - request = {"foo": 0, "bar": "1337"} - response = runner.client(ENDPT_NAME, ["0", "1337"]) - for p in farm.ports: - assert farm.counters()[p][ENDPT_NAME] == 1 - assert farm.counters()[p][ENDPT_NAME2] == 1 - assert farm.counters()[p][ENDPT_NAME3] == 1 - assert ENDPT_NAME in response - assert ENDPT_NAME2 in response - assert ENDPT_NAME3 in response - assert response["success"] is True - for h in farm.hosts: - # ENDPT2 - assert h in response[ENDPT_NAME2][ENDPT_NAME2] - assert "status" in response[ENDPT_NAME2][ENDPT_NAME2][h] - assert "reply" in response[ENDPT_NAME2][ENDPT_NAME2][h] - - assert response[ENDPT_NAME2][ENDPT_NAME2][h]["status"] == 200 - assert response[ENDPT_NAME2][ENDPT_NAME2][h]["reply"] == request - - # ENDPT3 - assert h in response[ENDPT_NAME3][ENDPT_NAME3] - assert "status" in response[ENDPT_NAME3][ENDPT_NAME3][h] - assert "reply" in response[ENDPT_NAME3][ENDPT_NAME3][h] - - assert response[ENDPT_NAME3][ENDPT_NAME3][h]["status"] == 200 - assert response[ENDPT_NAME3][ENDPT_NAME3][h]["reply"] == request - - for i in range(N_CALLS): - runner.client(ENDPT_NAME, ["0", "1337"]) - for p in farm.ports: - assert farm.counters()[p][ENDPT_NAME] == N_CALLS + 1 - assert farm.counters()[p][ENDPT_NAME2] == N_CALLS + 1 - assert farm.counters()[p][ENDPT_NAME3] == N_CALLS + 1 diff --git a/tests/coco_runner.py b/tests/coco_runner.py index c0a73ee3..fd272034 100644 --- a/tests/coco_runner.py +++ b/tests/coco_runner.py @@ -325,7 +325,14 @@ def start_daemon(self, *args): # even after it successfully fetches the port. sleep(0.2) - def client(self, *args, expect_failure=False, no_daemon=False, no_backend=False): + def client( + self, + *args, + expect_failure=False, + decode=False, + no_daemon=False, + no_backend=False, + ): """Invoke the coco client. Parameters @@ -335,12 +342,22 @@ def client(self, *args, expect_failure=False, no_daemon=False, no_backend=False) expect_failure : bool Controls whether a non-zero (failure) or zero (success) exit code will cause an assertion failure + decode : bool + If set to True, adds "--json" to the client command-line and then, + if the client exits sucessfully, JSON decodes the client output + and returns that. no_daemon : bool, optional If True, don't start the daemon before running the client. If the daemon is already running, this won't stop it. no_backend : bool If True, don't set the COCO_BACKEND envar. + + Returns + ------- + If `decode` is True, returns a JSON-decoded dict of the client output, + if possible. If `decode` is False (the default), or JSON-decoding + fails, returns the `click.Result` from the CliRunner. """ import traceback @@ -373,9 +390,16 @@ def client(self, *args, expect_failure=False, no_daemon=False, no_backend=False) else {"COCO_BACKEND": f"127.0.0.1:{self._daemon_port}"} ) + if decode: + args = ("--json", *args) + # Invoke result = runner.invoke(entry, args=args) + # Reset the coco client after the test. This needs to be done + # because the CliRunner doesn't run "coco" in standalone mode. + entry._coco_init = False + # Show traceback if one was created if ( result.exit_code @@ -394,9 +418,13 @@ def client(self, *args, expect_failure=False, no_daemon=False, no_backend=False) assert result.exit_code == 0 assert result.exception is None - # Reset the coco client after the test. This needs to be done - # because the CliRunner doesn't run "coco" in standalone mode. - entry._coco_init = False + # Attempt JSON decoding, if requested + if decode: + try: + result = json.loads(result.stdout) + except json.JSONDecodeError as e: + print(f"Failed to decode output: {e}") + pass # Return the result to the client return result diff --git a/tests/rest_server.py b/tests/rest_server.py index 8158eaf5..8d06db35 100644 --- a/tests/rest_server.py +++ b/tests/rest_server.py @@ -35,6 +35,12 @@ def record_hit(self, code, body, response=None): path = self.path.split("?", 1)[0] path = path.split("#", 1)[0] + # Decode body, if needed + try: + body = body.decode() + except AttributeError: + pass + # Pass back up to the RestServer instance self.server.rest_server.record_hit( path, Hit(self.path, self.command, body, int(code), response) @@ -79,29 +85,26 @@ def handle_route(self): # Split the path itself from the query query_split = self.path.split("?", 1) path = query_split[0] + query = "" if len(query_split) > 1: - body = "?" + query_split[1] + query = "?" + query_split[1] # Split the path from the fragment fragment_split = path.split("#", 1) path = fragment_split[0] if len(fragment_split) > 1: - if body: - body += "#" + fragment_split[1] + if query: + query += "#" + fragment_split[1] else: - body = "#" + fragment_split[1] + query = "#" + fragment_split[1] - # Read the body from a POST - if self.command == "POST": - body = self.rfile.read(int(self.headers["Content-Length"])) - else: - body = "" + # Read the body + body = self.rfile.read(int(self.headers["Content-Length"])) # If we're accepting all routes, reply with something generic if self.server.rest_server.any_route: response = {} - if self.command == "POST": - # For a POST, send back what we were given + if body: response["body"] = json.loads(body) # Add generic response data @@ -130,7 +133,7 @@ def handle_route(self): error_code = ( HTTPStatus.METHOD_NOT_ALLOWED if bad_method else HTTPStatus.NOT_FOUND ) - self.record_hit(error_code, body) + self.record_hit(error_code, body if body else query) self.send_error(error_code) return @@ -329,7 +332,10 @@ def _compare(a, b, equal, path=""): for hit in self.hits(route): # decode the request - received = json.loads(hit.request.decode()) + try: + received = json.loads(hit.request) + except json.JSONDecodeError: + received = hit.request # Return on first success result = _compare(received, sent, full) @@ -340,5 +346,5 @@ def _compare(a, b, equal, path=""): pytest.fail( f'Data not received by route "{route}": {result}\n' f"Expected\n {sent}\nReceived:\n " - "\n ".join([str(hit.request.decode()) for hit in self.hits(route)]) + "\n ".join([str(hit.request) for hit in self.hits(route)]) ) diff --git a/tests/test_call_coco.py b/tests/test_call_coco.py new file mode 100644 index 00000000..795e3e9b --- /dev/null +++ b/tests/test_call_coco.py @@ -0,0 +1,52 @@ +"""Test call forwarding to other coco endpoints,""" + + +def test_forward(coco_runner): + """Test that endpoints forward.""" + + coco_runner.add_targets("test", 2) + coco_runner.add_endpoint( + "proxy", + { + "call": {"coco": ["end", "end2"]}, + "group": "test", + "values": {"foo": "int", "bar": "str"}, + }, + ) + coco_runner.add_endpoint( + "end", {"group": "test", "values": {"foo": "int", "bar": "str"}} + ) + coco_runner.add_endpoint( + "end2", {"group": "test", "values": {"foo": "int", "bar": "str"}} + ) + + result = coco_runner.client("end", "--foo=123", "--bar=abc", decode=True) + assert set(result) == {"end", "queue_wait", "success"} + assert result["success"] is True + assert result["end"]["200"] == 2 + + result = coco_runner.client("end2", "--foo=456", "--bar=def", decode=True) + assert set(result) == {"end2", "queue_wait", "success"} + assert result["success"] is True + assert result["end2"]["200"] == 2 + + # Now via proxy + result = coco_runner.client("proxy", "--foo=789", "--bar=ghi", decode=True) + assert set(result) == {"end", "end2", "proxy", "queue_wait", "success"} + assert result["success"] is True + + # Check endpoints + assert result["proxy"]["200"] == 2 + assert result["end"]["end"]["200"] == 2 + assert result["end"]["success"] is True + assert result["end2"]["end2"]["200"] == 2 + assert result["end2"]["success"] is True + + # Check hits on the targets + for target in coco_runner.targets: + assert target.hit_count("/end") == 2 + target.assert_hit_received("/end", {"foo": 123, "bar": "abc"}) + target.assert_hit_received("/end", {"foo": 789, "bar": "ghi"}) + assert target.hit_count("/end2") == 2 + target.assert_hit_received("/end2", {"foo": 456, "bar": "def"}) + target.assert_hit_received("/end2", {"foo": 789, "bar": "ghi"}) From c0d79e246212944f2446763813c4faf3cbd5b099 Mon Sep 17 00:00:00 2001 From: "D. V. Wiebe" Date: Fri, 10 Jul 2026 12:52:31 -0700 Subject: [PATCH 02/11] test: recover test_checks Also: * default client method is now always GET * Re-implement --report-type (inadvertantly overlooked during client rewrite) * Some changes to rest_server fixture to make it more like the old rest_farm fixture. --- coco/client.py | 22 +-- old_tests/test_checks.py | 268 ----------------------------------- tests/coco_runner.py | 4 +- tests/conftest.py | 5 - tests/rest_server.py | 42 +++--- tests/test_checks.py | 291 +++++++++++++++++++++++++++++++++++++++ tests/test_client.py | 12 +- 7 files changed, 339 insertions(+), 305 deletions(-) delete mode 100644 old_tests/test_checks.py create mode 100644 tests/test_checks.py diff --git a/coco/client.py b/coco/client.py index 6a448327..815481aa 100644 --- a/coco/client.py +++ b/coco/client.py @@ -142,7 +142,10 @@ def _endpoint_params(self): # If there were no parameters, make one out of the name if len(param_decls) == 1: - param_decls.append("--" + name.lower().replace("_", "-")) + param_decls.append( + ("-" if len(name) == 1 else "--") + + name.lower().replace("_", "-") + ) # Add false-type params for bools if is_flag: @@ -293,11 +296,10 @@ def client_send_request( path : str Endpoint path. type : str, optional - HTTP request type. If `data` is provided, the default is "POST", - otherwise it's "GET". + HTTP request type. Defaults to "GET". data : Any - If `type` is not "GET", other keyword arguments to this function are - JSON-serialized and sent to the endpoint. + Other keyword arguments to this function are JSON-serialized and sent + to the endpoint. Returns ------- @@ -316,7 +318,7 @@ def client_send_request( # Determine HTTP command if type is None: - type_ = "post" if data else "get" + type_ = "get" else: type_ = type.lower() @@ -329,12 +331,12 @@ def client_send_request( client_options = ctx.obj["options"] + # Add report type + data["coco_report_type"] = client_options["report"] + # Short-circut for --show-call-only if client_options["show_call"]: - result = {"endpoint": url, "method": type_.upper()} - if type_ != "get": - result["data"] = data - return True, result + return True, {"endpoint": url, "method": type_.upper(), "data": data} silent = client_options["silent"] if silent: diff --git a/old_tests/test_checks.py b/old_tests/test_checks.py deleted file mode 100644 index d86509d3..00000000 --- a/old_tests/test_checks.py +++ /dev/null @@ -1,268 +0,0 @@ -"""Test endpoint calls triggered by failures.""" - -import multiprocessing -import random - -import pytest -from coco.test import coco_runner, endpoint_farm - -CONFIG = {"log_level": "INFO"} -ENDPOINTS = { - "type_check": { - "group": "test", - "values": {"ok": "bool"}, - "call": {"forward": {"name": "pong", "reply": {"type": {"ok": "bool"}}}}, - }, - "type_check_fail": { - "group": "test", - "values": {"ok": "int"}, - "call": {"forward": {"name": "pong", "reply": {"type": {"ok": "bool"}}}}, - }, - "value_check": { - "group": "test", - "values": {"ok": "bool"}, - "call": {"forward": {"name": "pong", "reply": {"value": {"ok": True}}}}, - }, - "identical_check": { - "group": "test", - "values": {"rand": "bool"}, - "call": {"forward": {"name": "rand", "reply": {"identical": ["rand"]}}}, - }, - "pong": {"group": "test"}, - "rand": {"group": "test"}, - # For before check - "bvalue_check": { - "group": "test", - "values": {"ok": "bool"}, - "call": {"forward": None}, - "before": {"name": "pong", "reply": {"value": {"ok": True}}}, - }, - "avalue_check": { - "group": "test", - "values": {"ok": "bool"}, - "call": {"forward": None}, - "after": {"name": "pong", "reply": {"value": {"ok": True}}}, - }, - "save_to_state": { - "group": "test", - "values": {"a": "bool", "b": "bool"}, - "call": {"forward": None}, - "save_state": "fo/bar", - }, - "save_to_state_fu": { - "group": "test", - "values": {"b": "bool"}, - "call": {"forward": None}, - "save_state": "fu/bar", - }, - "state_check_path": { - "group": "test", - "values": {"b": "bool"}, - "call": {"forward": {"name": "pong", "reply": {"state": "fu/bar"}}}, - }, - "state_check_values": { - "group": "test", - "values": {"a": "bool", "b": "bool"}, - "call": { - "forward": { - "name": "pong", - "reply": {"state": {"a": "fo/bar/a", "b": "fo/bar/b"}}, - } - }, - }, -} -N_CALLS = 2 - - -def callback(data): - """Reply with the incoming json request.""" - return data - - -class RandCallback: - """ - Reply with a not repeating random number if rand=True received, - otherwise reply with a fixed number. - """ - - def __init__(self): - self.fixed_num = multiprocessing.Value("d", 0.0) - - def __call__(self, data): - if data["rand"]: - rand = random.random() - - with self.fixed_num.get_lock(): - self.fixed_num.value += rand - return {"rand": rand} - - return {"rand": self.fixed_num.value} - - -rand_callback = RandCallback() - -N_HOSTS = 2 -CALLBACKS = {"pong": callback, "rand": rand_callback} - - -@pytest.fixture -def farm(): - """Create a coco runner.""" - return endpoint_farm.Farm(N_HOSTS, CALLBACKS) - - -@pytest.fixture -def runner(farm): - """Create an endpoint test farm.""" - CONFIG["groups"] = {"test": farm.hosts} - with coco_runner.Runner(CONFIG, ENDPOINTS) as runner: - yield runner - - -def test_forward(farm, runner): - """Test coco's forward reply check.""" - # Test failed type check - response = runner.client("type_check_fail", ["1"]) - for p in farm.ports: - assert farm.counters()[p]["pong"] == 1 - - # Check failure report - assert response["success"] is False - failed_host = list(response["failed_checks"]["pong"].keys()) - assert len(failed_host) == N_HOSTS - reply = response["failed_checks"]["pong"][failed_host[0]]["reply"] - assert reply["type"] == ["ok"] - - # Test passing type check - response = runner.client("type_check", ["False"]) - for p in farm.ports: - assert farm.counters()[p]["pong"] == 2 - - # Check failure report - assert response["success"] is True - assert "failed_checks" not in response - - # Test failed value check - response = runner.client("value_check", ["False"]) - for p in farm.ports: - assert farm.counters()[p]["pong"] == 3 - assert response["success"] is False - - response = runner.client("value_check", ["False"]) - for p in farm.ports: - assert farm.counters()[p]["pong"] == 4 - - # Check failure report - assert response["success"] is False - failed_host = list(response["failed_checks"]["pong"].keys()) - assert len(failed_host) == N_HOSTS - reply = response["failed_checks"]["pong"][failed_host[0]]["reply"] - assert reply["value"] == ["ok"] - - # Test passing value check - response = runner.client("value_check", ["True"]) - for p in farm.ports: - assert farm.counters()[p]["pong"] == 5 - - # Check failure report - assert response["success"] is True - assert "failed_checks" not in response - - # Test failed identical check - response = runner.client("identical_check", ["True"]) - for p in farm.ports: - assert farm.counters()[p]["rand"] == 1 - - # Check failure report - assert response["success"] is False - failed_host = list(response["failed_checks"]["rand"].keys()) - assert len(failed_host) == N_HOSTS - reply = response["failed_checks"]["rand"][failed_host[0]]["reply"] - assert reply["not_identical"] == ["all"] - - # Test passing identical check - response = runner.client("identical_check", ["False"]) - for p in farm.ports: - assert farm.counters()[p]["rand"] == 2 - - # Check failure report - assert response["success"] is True - assert "failed_checks" not in response - - # Test failed check on before - response = runner.client("bvalue_check", ["True"]) - for p in farm.ports: - assert farm.counters()[p]["pong"] == 6 - - # Check failure report - assert response["success"] is False - failed_host = list(response["pong"]["failed_checks"]["pong"].keys()) - assert len(failed_host) == N_HOSTS - reply = response["pong"]["failed_checks"]["pong"][failed_host[0]]["reply"] - assert reply["missing"] == ["ok"] - - # Test failed check on after - response = runner.client("avalue_check", ["True"]) - for p in farm.ports: - assert farm.counters()[p]["pong"] == 7 - - # Check failure report - assert response["success"] is False - failed_host = list(response["pong"]["failed_checks"]["pong"].keys()) - assert len(failed_host) == N_HOSTS - reply = response["pong"]["failed_checks"]["pong"][failed_host[0]]["reply"] - assert reply["missing"] == ["ok"] - - # Test state checks - # ------------------------------------------------------------------- - # First without setting it (will always fail) - response = runner.client("state_check_path", ["True"]) - assert response["success"] is False - failed_host = list(response["failed_checks"]["pong"].keys()) - assert len(failed_host) == N_HOSTS - - response = runner.client("state_check_path", ["False"]) - assert response["success"] is False - failed_host = list(response["failed_checks"]["pong"].keys()) - assert len(failed_host) == N_HOSTS - - # Set the state - response = runner.client("save_to_state_fu", ["False"]) - assert response["success"] is True - - # Test again - response = runner.client("state_check_path", ["True"]) - assert response["success"] is False - failed_host = list(response["failed_checks"]["pong"].keys()) - assert len(failed_host) == N_HOSTS - - response = runner.client("state_check_path", ["False"]) - assert response["success"] is True - assert "failed_checks" not in response - - # The same with multiple paths to compare between state and reply - # ------------------------------------------------------------------- - # First without setting it (will always fail) - response = runner.client("state_check_values", ["True", "True"]) - assert response["success"] is False - failed_host = list(response["failed_checks"]["pong"].keys()) - assert len(failed_host) == N_HOSTS - - response = runner.client("state_check_values", ["False", "False"]) - assert response["success"] is False - failed_host = list(response["failed_checks"]["pong"].keys()) - assert len(failed_host) == N_HOSTS - - # Set the state - response = runner.client("save_to_state", ["False", "False"]) - assert response["success"] is True - - # Test again - response = runner.client("state_check_values", ["False", "False"]) - assert response["success"] is True - assert "failed_checks" not in response - - response = runner.client("state_check_values", ["False", "True"]) - assert response["success"] is False - failed_host = list(response["failed_checks"]["pong"].keys()) - assert len(failed_host) == N_HOSTS diff --git a/tests/coco_runner.py b/tests/coco_runner.py index fd272034..a103af96 100644 --- a/tests/coco_runner.py +++ b/tests/coco_runner.py @@ -181,7 +181,7 @@ def add_config(self, **extra_config): # Merge in config self.config = config.merge_dict_tree(self.config, extra_config) - def add_targets(self, group, count): + def add_targets(self, group, count, callback=None): """Add `count` target rest servers to the group `group`. The targets are created using the rest_server test fixture. @@ -200,7 +200,7 @@ def add_targets(self, group, count): targets = [] for _ in range(count): server = self.rest_server() - server.accept_all() + server.accept_all(callback=callback) server.start() # Remember it so we can stop it later diff --git a/tests/conftest.py b/tests/conftest.py index 6f6c5c35..b5e85012 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,5 @@ """Common test fixtures""" -import json import traceback import pytest @@ -89,10 +88,6 @@ def mock_comet(rest_server): def _register_state(route, body): """Pretend to be comet's /register-state endpoint.""" - - # Decode body - body = json.loads(body) - return {"result": "success", "request": "get_state", "hash": body["hash"]} # Create a mock broker diff --git a/tests/rest_server.py b/tests/rest_server.py index 8d06db35..013fe1fa 100644 --- a/tests/rest_server.py +++ b/tests/rest_server.py @@ -99,17 +99,27 @@ def handle_route(self): query = "#" + fragment_split[1] # Read the body - body = self.rfile.read(int(self.headers["Content-Length"])) + if self.headers["Content-Length"]: + body = self.rfile.read(int(self.headers["Content-Length"])) + if isinstance(body, bytes): + body = body.decode() + body = json.loads(body) + else: + body = {} - # If we're accepting all routes, reply with something generic + # Handle accepting any route if self.server.rest_server.any_route: - response = {} - if body: - response["body"] = json.loads(body) + if self.server.rest_server.any_callback: + response = self.server.rest_server.any_callback(path, body) + else: + # If no callback, generate a generic response + response = {} + if body: + response["body"] = body - # Add generic response data - response["path"] = path - response["result"] = "success" + # Add generic response data + response["path"] = path + response["result"] = "success" # Return the response. self.send_json(body, response) @@ -121,7 +131,7 @@ def handle_route(self): if route.method == self.command: # Handle the route if route.callback: - response = route.callback(route, body.decode()) + response = route.callback(route, body) else: response = route.response # Return the response. @@ -167,6 +177,9 @@ def __init__(self): # If True, all routes are accepted self.any_route = False + # If accepting all routes, this will be called to create the return value + self.any_callback = None + # List of routes added with add_route self.routes = [] @@ -179,9 +192,10 @@ def __init__(self): # Is the server running? self.running = False - def accept_all(self): + def accept_all(self, callback=None): """Set up this rest server to accept any endpoint.""" self.any_route = True + self.any_callback = callback def add_route(self, path, method="GET", callback=None, response=None): """Add a route to the server. @@ -331,14 +345,8 @@ def _compare(a, b, equal, path=""): return None for hit in self.hits(route): - # decode the request - try: - received = json.loads(hit.request) - except json.JSONDecodeError: - received = hit.request - # Return on first success - result = _compare(received, sent, full) + result = _compare(hit.request, sent, full) if not result: return diff --git a/tests/test_checks.py b/tests/test_checks.py new file mode 100644 index 00000000..627e531e --- /dev/null +++ b/tests/test_checks.py @@ -0,0 +1,291 @@ +"""Test endpoint calls triggered by failures.""" + +import random +from threading import Lock + +fixed_num = 0 +lock = Lock() + + +def callback(path, body): + """Used by the rest_server to generate data + to send back to cocod.""" + + global fixed_num + + if path == "/rand": + # If the body has a True "rand" value, return + # a random number. Otherwise return a fixed number. + with lock: + if body["rand"]: + rand = random.random() + + fixed_num += rand + return {"rand": rand} + return {"rand": fixed_num} + + # Otherwise, just return body + return body + + +def test_forward(coco_runner): + """Test coco's forward reply check.""" + + # Number of targets + N_HOSTS = 2 + + # Create rest farm with the above callback + coco_runner.add_targets("test", N_HOSTS, callback=callback) + + # Add endpoints + coco_runner.add_endpoint( + "type_check", + { + "group": "test", + "values": {"ok": "bool"}, + "call": {"forward": {"name": "pong", "reply": {"type": {"ok": "bool"}}}}, + }, + ) + coco_runner.add_endpoint( + "type_check_fail", + { + "group": "test", + "values": {"ok": "int"}, + "call": {"forward": {"name": "pong", "reply": {"type": {"ok": "bool"}}}}, + }, + ) + coco_runner.add_endpoint( + "value_check", + { + "group": "test", + "values": {"ok": "bool"}, + "call": {"forward": {"name": "pong", "reply": {"value": {"ok": True}}}}, + }, + ) + coco_runner.add_endpoint( + "identical_check", + { + "group": "test", + "values": {"rand": "bool"}, + "call": {"forward": {"name": "rand", "reply": {"identical": ["rand"]}}}, + }, + ) + coco_runner.add_endpoint("pong", {"group": "test"}) + coco_runner.add_endpoint("rand", {"group": "test"}) + + # For before check + coco_runner.add_endpoint( + "bvalue_check", + { + "group": "test", + "values": {"ok": "bool"}, + "call": {"forward": None}, + "before": {"name": "pong", "reply": {"value": {"ok": True}}}, + }, + ) + coco_runner.add_endpoint( + "avalue_check", + { + "group": "test", + "values": {"ok": "bool"}, + "call": {"forward": None}, + "after": {"name": "pong", "reply": {"value": {"ok": True}}}, + }, + ) + coco_runner.add_endpoint( + "save_to_state", + { + "group": "test", + "values": {"a": "bool", "b": "bool"}, + "call": {"forward": None}, + "save_state": "fo/bar", + }, + ) + coco_runner.add_endpoint( + "save_to_state_fu", + { + "group": "test", + "values": {"b": "bool"}, + "call": {"forward": None}, + "save_state": "fu/bar", + }, + ) + coco_runner.add_endpoint( + "state_check_path", + { + "group": "test", + "values": {"b": "bool"}, + "call": {"forward": {"name": "pong", "reply": {"state": "fu/bar"}}}, + }, + ) + coco_runner.add_endpoint( + "state_check_values", + { + "group": "test", + "values": {"a": "bool", "b": "bool"}, + "call": { + "forward": { + "name": "pong", + "reply": {"state": {"a": "fo/bar/a", "b": "fo/bar/b"}}, + } + }, + }, + ) + + # Test failed type check + response = coco_runner.client( + "-r", "FULL", "type_check_fail", "--ok=1", decode=True + ) + for p in coco_runner.targets: + p.assert_hit_received("/pong", {"ok": 1}) + + # Check failure report + assert response["success"] is False + failed_host = list(response["failed_checks"]["pong"]) + assert len(failed_host) == N_HOSTS + reply = response["failed_checks"]["pong"][failed_host[0]]["reply"] + assert reply["type"] == ["ok"] + + # Test passing type check + response = coco_runner.client("type_check", "--no-ok", decode=True) + for p in coco_runner.targets: + assert p.hit_count("/pong") == 2 + + # Check failure report + assert response["success"] is True + assert "failed_checks" not in response + + # Test failed value check + response = coco_runner.client("value_check", "--no-ok", decode=True) + for p in coco_runner.targets: + assert p.hit_count("/pong") == 3 + assert response["success"] is False + + response = coco_runner.client("-r", "FULL", "value_check", "--no-ok", decode=True) + for p in coco_runner.targets: + assert p.hit_count("/pong") == 4 + + # Check failure report + assert response["success"] is False + failed_host = list(response["failed_checks"]["pong"]) + assert len(failed_host) == N_HOSTS + reply = response["failed_checks"]["pong"][failed_host[0]]["reply"] + assert reply["value"] == ["ok"] + + # Test passing value check + response = coco_runner.client("value_check", "--ok", decode=True) + for p in coco_runner.targets: + assert p.hit_count("/pong") == 5 + + # Check failure report + assert response["success"] is True + assert "failed_checks" not in response + + # Test failed identical check + response = coco_runner.client( + "-r", "FULL", "identical_check", "--rand", decode=True + ) + for p in coco_runner.targets: + assert p.hit_count("/rand") == 1 + + # Check failure report + assert response["success"] is False + failed_host = list(response["failed_checks"]["rand"].keys()) + assert len(failed_host) == N_HOSTS + reply = response["failed_checks"]["rand"][failed_host[0]]["reply"] + assert reply["not_identical"] == ["all"] + + # Test passing identical check + response = coco_runner.client("identical_check", "--no-rand", decode=True) + for p in coco_runner.targets: + assert p.hit_count("/rand") == 2 + + # Check failure report + assert response["success"] is True + assert "failed_checks" not in response + + # Test failed check on before + response = coco_runner.client("-r", "FULL", "bvalue_check", "--ok", decode=True) + for p in coco_runner.targets: + assert p.hit_count("/pong") == 6 + + # Check failure report + assert response["success"] is False + failed_host = list(response["pong"]["failed_checks"]["pong"].keys()) + assert len(failed_host) == N_HOSTS + reply = response["pong"]["failed_checks"]["pong"][failed_host[0]]["reply"] + assert reply["missing"] == ["ok"] + + # Test failed check on after + response = coco_runner.client("-r", "FULL", "avalue_check", "--ok", decode=True) + for p in coco_runner.targets: + assert p.hit_count("/pong") == 7 + + # Check failure report + assert response["success"] is False + failed_host = list(response["pong"]["failed_checks"]["pong"].keys()) + assert len(failed_host) == N_HOSTS + reply = response["pong"]["failed_checks"]["pong"][failed_host[0]]["reply"] + assert reply["missing"] == ["ok"] + + # Test state checks + # ------------------------------------------------------------------- + # First without setting it (will always fail) + response = coco_runner.client("-r", "FULL", "state_check_path", "-b", decode=True) + assert response["success"] is False + failed_host = list(response["failed_checks"]["pong"].keys()) + assert len(failed_host) == N_HOSTS + + response = coco_runner.client( + "-r", "FULL", "state_check_path", "--no-b", decode=True + ) + assert response["success"] is False + failed_host = list(response["failed_checks"]["pong"].keys()) + assert len(failed_host) == N_HOSTS + + # Set the state + response = coco_runner.client("save_to_state_fu", "--no-b", decode=True) + assert response["success"] is True + + # Test again + response = coco_runner.client("-r", "FULL", "state_check_path", "-b", decode=True) + assert response["success"] is False + failed_host = list(response["failed_checks"]["pong"].keys()) + assert len(failed_host) == N_HOSTS + + response = coco_runner.client("state_check_path", "--no-b", decode=True) + assert response["success"] is True + assert "failed_checks" not in response + + # The same with multiple paths to compare between state and reply + # ------------------------------------------------------------------- + # First without setting it (will always fail) + response = coco_runner.client( + "-r", "FULL", "state_check_values", "-a", "-b", decode=True + ) + assert response["success"] is False + failed_host = list(response["failed_checks"]["pong"].keys()) + assert len(failed_host) == N_HOSTS + + response = coco_runner.client( + "-r", "FULL", "state_check_values", "--no-a", "--no-b", decode=True + ) + assert response["success"] is False + failed_host = list(response["failed_checks"]["pong"].keys()) + assert len(failed_host) == N_HOSTS + + # Set the state + response = coco_runner.client("save_to_state", "--no-a", "--no-b", decode=True) + assert response["success"] is True + + # Test again + response = coco_runner.client("state_check_values", "--no-a", "--no-b", decode=True) + assert response["success"] is True + assert "failed_checks" not in response + + response = coco_runner.client( + "-r", "FULL", "state_check_values", "--no-a", "-b", decode=True + ) + assert response["success"] is False + failed_host = list(response["failed_checks"]["pong"].keys()) + assert len(failed_host) == N_HOSTS diff --git a/tests/test_client.py b/tests/test_client.py index bc8fa766..9b9690ec 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -51,6 +51,7 @@ def test_style(coco_runner): expected_result = { "endpoint": f"http://127.0.0.1:{coco_runner.port}/nop", "method": "GET", + "data": {"coco_report_type": "CODES_OVERVIEW"}, } # Default is yaml @@ -91,6 +92,7 @@ def test_show_call(coco_runner): assert result == { "endpoint": f"http://127.0.0.1:{coco_runner.port}/nop", "method": "GET", + "data": {"coco_report_type": "CODES_OVERVIEW"}, } # A local endpoint @@ -101,8 +103,12 @@ def test_show_call(coco_runner): ) assert result == { "endpoint": f"http://127.0.0.1:{coco_runner.port}/update-blocklist", - "method": "POST", - "data": {"command": "add", "hosts": ["HOST"]}, + "method": "GET", + "data": { + "command": "add", + "hosts": ["HOST"], + "coco_report_type": "CODES_OVERVIEW", + }, } # coco config -- the endpoint here is not called, but --show-call-only @@ -314,5 +320,5 @@ def test_value_handling(coco_runner): assert target.hit_count("/endpoint") == 1 hit = target.hits("/endpoint")[0] assert hit.method == "POST" - assert json.loads(hit.request) == data + assert hit.request == data assert hit.response == {"path": "/endpoint", "result": "success", "body": data} From 774ea87062cd4f75281a619e5a124e981f972d69 Mon Sep 17 00:00:00 2001 From: "D. V. Wiebe" Date: Fri, 10 Jul 2026 14:06:37 -0700 Subject: [PATCH 03/11] test: recover test_forward Also: * Add a `call_endpoint` method to `coco_runner` to simplify direct access to the daemon (i.e. without using the client). --- old_tests/test_forward.py | 118 -------------------------------------- tests/coco_runner.py | 67 +++++++++++++++++++++- tests/rest_server.py | 14 +++-- tests/test_forward.py | 93 ++++++++++++++++++++++++++++++ 4 files changed, 166 insertions(+), 126 deletions(-) delete mode 100644 old_tests/test_forward.py create mode 100644 tests/test_forward.py diff --git a/old_tests/test_forward.py b/old_tests/test_forward.py deleted file mode 100644 index 67cbc0b8..00000000 --- a/old_tests/test_forward.py +++ /dev/null @@ -1,118 +0,0 @@ -"""Test basic endpoint call forwarding.""" - -import pytest -import requests -from coco.test import coco_runner, endpoint_farm - -ENDPT_NAME = "test" -PORT = 12055 -CONFIG = {"log_level": "INFO", "port": PORT} -ENDPOINTS = { - ENDPT_NAME: { - "group": "test", - "report_latencies": True, - "values": {"foo": "int", "bar": "str"}, - } -} -N_CALLS = 2 - - -def callback(data): - """Reply with the incoming json request.""" - return data - - -N_HOSTS = 2 -CALLBACKS = {ENDPT_NAME: callback} - - -@pytest.fixture(scope="module") -def farm(): - """Create a coco runner.""" - return endpoint_farm.Farm(N_HOSTS, CALLBACKS) - - -@pytest.fixture(scope="module") -def runner(farm): - """Create an endpoint test farm.""" - CONFIG["groups"] = {"test": farm.hosts} - with coco_runner.Runner(CONFIG, ENDPOINTS) as runner: - yield runner - - -def test_forward(farm, runner): - """Test if a request gets forwarded to an external endpoint.""" - request = {"foo": 0, "bar": "1337"} - response = runner.client(ENDPT_NAME, ["0", "1337"]) - - for p in farm.ports: - assert farm.counters()[p][ENDPT_NAME] == 1 - assert ENDPT_NAME in response - for h in farm.hosts: - assert h in response[ENDPT_NAME] - assert "status" in response[ENDPT_NAME][h] - assert "reply" in response[ENDPT_NAME][h] - - assert response[ENDPT_NAME][h]["status"] == 200 - assert response[ENDPT_NAME][h]["reply"] == request - - for i in range(N_CALLS): - runner.client(ENDPT_NAME, ["0", "1337"]) - for p in farm.ports: - assert farm.counters()[p][ENDPT_NAME] == N_CALLS + 1 - - -def test_wrong_vars(farm, runner): - request = {"foo": "dfg", "bar": 1337} - response = requests.get(f"http://localhost:{PORT}/{ENDPT_NAME}", json=request) - assert response.status_code == 400 - - request = {"foo": 1337} - response = requests.get(f"http://localhost:{PORT}/{ENDPT_NAME}", json=request) - assert response.status_code == 400 - - -def test_url_args(farm, runner): - """Test if URL arguments get forwarded to an external endpoint.""" - request = {"foo": 0, "bar": "1337"} - request_full = request.copy() - request_full.update({"coco_report_type": "FULL"}) - params = {"cat": "1", "hat": "rat"} - query_str = "&".join([f"{k}={params[k]}" for k in params]) - - response = requests.get( - f"http://localhost:{PORT}/{ENDPT_NAME}?{query_str}", json=request_full - ) - assert response.status_code == 200 - response = response.json() - - assert ENDPT_NAME in response - for h in farm.hosts: - assert h in response[ENDPT_NAME] - assert "status" in response[ENDPT_NAME][h] - assert "reply" in response[ENDPT_NAME][h] - assert "params" in response[ENDPT_NAME][h]["reply"] - assert ( - "latency" in response[ENDPT_NAME][h] - ) # Check that per-host latencies are returned - - request.update({"params": params}) - assert response[ENDPT_NAME][h]["status"] == 200 - assert response[ENDPT_NAME][h]["reply"] == request - - -def test_latency_stats(farm, runner): - """Test if latency stats are returned from external endpoint""" - request = {"foo": 0, "bar": "1337"} - params = {"cat": "1", "hat": "rat"} - query_str = "&".join([f"{k}={params[k]}" for k in params]) - - response = requests.get( - f"http://localhost:{PORT}/{ENDPT_NAME}?{query_str}", json=request - ) - assert response.status_code == 200 - response = response.json() - - print(response) - - assert "latency_stats" in response[ENDPT_NAME] diff --git a/tests/coco_runner.py b/tests/coco_runner.py index a103af96..b1d7a7da 100644 --- a/tests/coco_runner.py +++ b/tests/coco_runner.py @@ -25,12 +25,14 @@ def testN(coco_runner): coco_runner.client("callN") """ +import asyncio import json import multiprocessing import socket import threading from time import sleep +import aiohttp import fakeredis import pytest import yaml @@ -325,6 +327,65 @@ def start_daemon(self, *args): # even after it successfully fetches the port. sleep(0.2) + def call_endpoint( + self, + endpoint: str, + method: str = "GET", + data: dict | None = None, + query: str | None = None, + ): + """Call a daemon endpoint directly. + + i.e. without using the client. Starts the daemon if it isn't + already running. + + Parameters + ---------- + endpoint : str + Endpoint to call + method : str + HTTP method to use. The only values honoured are "GET" and "POST". + Other values are taken to mean "GET" (the default). + data : dict, optional + A dict of data to serialize to send to the daemon. + query : str, optional + URL query, if any (i.e. the part after '?' in the URL). Should be + URI-encoded. + + Returns + ------- + aiothhtp.ClientResponse + The response + Any: + The response body. If JSON-encoded, it will be decoded. + """ + + # Can't be called after stop() + if self._daemon_result: + raise RuntimeError("called after daemon stop.") + self.start_daemon() + + # Build the URL + url = f"http://127.0.0.1:{self._daemon_port}/{endpoint}" + if query: + url += "?" + query + + async def _get_response(method, url): + async with aiohttp.ClientSession() as session: + if method == "POST": + method = session.post + else: + method = session.get + + async with method(url, json=data) as response: + try: + body = await response.json() + except aiohttp.ContentTypeError: + body = await response.text() + return (response, body) + + return asyncio.run(_get_response(method, url)) + def client( self, *args, @@ -412,10 +473,12 @@ def client( print(result.output) if expect_failure: - assert result.exit_code != 0 + assert result.exit_code != 0, "Client failed to exit with error" assert type(result.exception) is SystemExit else: - assert result.exit_code == 0 + assert result.exit_code == 0, ( + f"Client exited with error: {result.exit_code}" + ) assert result.exception is None # Attempt JSON decoding, if requested diff --git a/tests/rest_server.py b/tests/rest_server.py index 013fe1fa..2b285887 100644 --- a/tests/rest_server.py +++ b/tests/rest_server.py @@ -85,18 +85,16 @@ def handle_route(self): # Split the path itself from the query query_split = self.path.split("?", 1) path = query_split[0] - query = "" + query = None if len(query_split) > 1: - query = "?" + query_split[1] + query = query_split[1] # Split the path from the fragment fragment_split = path.split("#", 1) path = fragment_split[0] + fragment = None if len(fragment_split) > 1: - if query: - query += "#" + fragment_split[1] - else: - query = "#" + fragment_split[1] + fragment = fragment_split[1] # Read the body if self.headers["Content-Length"]: @@ -116,6 +114,10 @@ def handle_route(self): response = {} if body: response["body"] = body + if query: + response["query"] = query + if fragment: + response["fragment"] = fragment # Add generic response data response["path"] = path diff --git a/tests/test_forward.py b/tests/test_forward.py new file mode 100644 index 00000000..f509517e --- /dev/null +++ b/tests/test_forward.py @@ -0,0 +1,93 @@ +"""Test basic endpoint call forwarding.""" + +import pytest + + +@pytest.fixture +def runner(coco_runner): + """coco_runner with pre-made endpoint. + + (And targets) + """ + + coco_runner.add_targets("test", 2) + coco_runner.add_endpoint( + "test", + { + "group": "test", + "report_latency": True, + "values": {"foo": "int", "bar": "str"}, + }, + ) + return coco_runner + + +def test_forward(runner): + """Test if a request gets forwarded to an external endpoint.""" + request = {"foo": 0, "bar": "1337"} + response = runner.client("-r", "FULL", "test", "--foo=0", "--bar=1337", decode=True) + + assert "test" in response + for t in runner.targets: + assert t.hit_count("/test") == 1 + h = f"http://127.0.0.1:{t.port}/" + assert h in response["test"] + assert "status" in response["test"][h] + assert "reply" in response["test"][h] + + assert response["test"][h]["status"] == 200 + assert response["test"][h]["reply"]["body"] == request + + runner.client("test", "--foo=0", "--bar=1337") + runner.client("test", "--foo=0", "--bar=1337") + for t in runner.targets: + assert t.hit_count("/test") == 3 + + +def test_wrong_vars(runner): + request = {"foo": "dfg", "bar": 1337} + response, _ = runner.call_endpoint("test", data=request) + assert response.status == 400 + + request = {"foo": 1337} + response, _ = runner.call_endpoint("test", data=request) + assert response.status == 400 + + +def test_url_args(runner): + """Test if URL arguments get forwarded to an external endpoint.""" + + request = {"foo": 0, "bar": "1337"} + request_full = request.copy() + request_full.update({"coco_report_type": "FULL"}) + params = {"cat": "1", "hat": "rat"} + query_str = "&".join([f"{k}={params[k]}" for k in params]) + + response, json = runner.call_endpoint("test", data=request_full, query=query_str) + assert response.status == 200 + + assert "test" in json + for t in runner.targets: + h = f"http://127.0.0.1:{t.port}/" + assert h in json["test"] + assert "status" in json["test"][h] + assert "reply" in json["test"][h] + assert "query" in json["test"][h]["reply"] + assert ( + "latency" in json["test"][h] + ) # Check that per-host latencies are returned + + assert json["test"][h]["status"] == 200 + assert json["test"][h]["reply"] == { + "body": request, + "path": "/test", + "query": query_str, + "result": "success", + } + + +def test_latency_stats(runner): + """Test if latency stats are returned from external endpoint""" + + response = runner.client("test", "--foo=0", "--bar=1337", decode=True) + assert "latency_stats" in response["test"] From ea896e49c6239cf6c6d774ea2470f87acb78d064 Mon Sep 17 00:00:00 2001 From: "D. V. Wiebe" Date: Fri, 10 Jul 2026 14:43:37 -0700 Subject: [PATCH 04/11] test: recover test_on_failure.py --- old_tests/test_on_failure.py | 114 ----------------------------------- tests/test_on_failure.py | 92 ++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 114 deletions(-) delete mode 100644 old_tests/test_on_failure.py create mode 100644 tests/test_on_failure.py diff --git a/old_tests/test_on_failure.py b/old_tests/test_on_failure.py deleted file mode 100644 index e36ad743..00000000 --- a/old_tests/test_on_failure.py +++ /dev/null @@ -1,114 +0,0 @@ -"""Test endpoint calls triggered by failures.""" - -import multiprocessing - -import pytest -from coco.test import coco_runner, endpoint_farm - -CONFIG = {"log_level": "INFO"} -ENDPOINTS = { - "call_single": { - "group": "test", - "call": { - "forward": { - "name": "status", - "reply": {"type": {"ok": "bool"}}, - "on_failure": {"call_single_host": "restart"}, - } - }, - }, - "call_all": { - "group": "test", - "call": { - "forward": { - "name": "status", - "reply": {"type": {"ok": "bool"}}, - "on_failure": {"call": "restart"}, - } - }, - }, - "status": {"group": "test"}, - "restart": {"group": "test"}, -} -N_CALLS = 2 - - -def callback(data): - """Reply with the incoming json request.""" - return data - - -class FailStatus: - """Send an invalid reply after a fixed number of requests.""" - - def __init__(self, fail_on=0): - self.fail_on = fail_on - self.count = multiprocessing.Value("i", 0) - - def callback(self, data): - """Return invalid reply when specified.""" - if self.count.value == self.fail_on: - return {"not_ok": True} - - with self.count.get_lock(): - self.count.value += 1 - return {"ok": True} - - -N_HOSTS = 2 -STATUS_FAILER = FailStatus(1) -CALLBACKS = {"restart": callback, "status": STATUS_FAILER.callback} - - -@pytest.fixture -def farm(): - """Create a coco runner.""" - return endpoint_farm.Farm(N_HOSTS, CALLBACKS) - - -@pytest.fixture -def runner(farm): - """Create an endpoint test farm.""" - CONFIG["groups"] = {"test": farm.hosts} - with coco_runner.Runner(CONFIG, ENDPOINTS) as runner: - yield runner - - -def test_on_reply(farm, runner): - """Test coco's on_failure option.""" - - # Test call on failure - response = runner.client("call_all") - for p in farm.ports: - assert farm.counters()[p]["status"] == 1 - assert ( - farm.counters()[p]["restart"] == 1 - ) # This key is not in the Manager dict? - - # Check failure report - failed_host = list(response["failed_checks"]["status"].keys()) - assert len(failed_host) == 1 - reply = response["failed_checks"]["status"][failed_host[0]]["reply"] - assert reply["missing"] == ["ok"] - - # reset count - with STATUS_FAILER.count.get_lock(): - STATUS_FAILER.count.value = 0 - - # Test call_single_host - response = runner.client("call_single") - - # Check failure report - failed_host = list(response["failed_checks"]["status"].keys()) - assert len(failed_host) == 1 - reply = response["failed_checks"]["status"][failed_host[0]]["reply"] - assert reply["missing"] == ["ok"] - - # Check only failed host called restart - for p in farm.ports: - assert farm.counters()[p]["status"] == 2 - # only second host should have failed - if p == int(failed_host[0].strip("http://").split(":")[1]): - assert farm.counters()[p]["restart"] == 2 - else: - assert farm.counters()[p]["restart"] == 1 diff --git a/tests/test_on_failure.py b/tests/test_on_failure.py new file mode 100644 index 00000000..bcb99cf4 --- /dev/null +++ b/tests/test_on_failure.py @@ -0,0 +1,92 @@ +"""Test endpoint calls triggered by failures.""" + +from threading import Lock + +count = 0 +lock = Lock() + + +def callback(path, body): + """Reply with the incoming json request.""" + global count + + # for /restart, just return input + if path == "/restart": + return body + + # When count gets to 1, return an invalid reply + with lock: + if count > 0: + return {"not_ok": True} + count += 1 + return {"ok": True} + + +def test_on_reply(coco_runner): + """Test coco's on_failure option.""" + global count + + N_HOSTS = 2 + coco_runner.add_targets("test", N_HOSTS, callback=callback) + coco_runner.add_endpoint( + "call_single", + { + "group": "test", + "call": { + "forward": { + "name": "status", + "reply": {"type": {"ok": "bool"}}, + "on_failure": {"call_single_host": "restart"}, + } + }, + }, + ) + coco_runner.add_endpoint( + "call_all", + { + "group": "test", + "call": { + "forward": { + "name": "status", + "reply": {"type": {"ok": "bool"}}, + "on_failure": {"call": "restart"}, + } + }, + }, + ) + coco_runner.add_endpoint("status", {"group": "test"}) + coco_runner.add_endpoint("restart", {"group": "test"}) + + # Test call on failure + response = coco_runner.client("-r", "FULL", "call_all", decode=True) + for t in coco_runner.targets: + assert t.hit_count("/status") == 1 + assert t.hit_count("/restart") == 1 + + # Check failure report + failed_host = list(response["failed_checks"]["status"].keys()) + assert len(failed_host) == 1 + reply = response["failed_checks"]["status"][failed_host[0]]["reply"] + assert reply["missing"] == ["ok"] + + # reset count + with lock: + count = 0 + + # Test call_single_host + response = coco_runner.client("-r", "FULL", "call_single", decode=True) + + # Check failure report + failed_host = list(response["failed_checks"]["status"].keys()) + assert len(failed_host) == 1 + reply = response["failed_checks"]["status"][failed_host[0]]["reply"] + assert reply["missing"] == ["ok"] + + # Check only failed host called restart + for t in coco_runner.targets: + assert t.hit_count("/status") == 2 + # only second host should have failed + if t.port == int(failed_host[0].strip("http://").split(":")[1]): + assert t.hit_count("/restart") == 2 + else: + assert t.hit_count("/restart") == 1 From c3141ffade2b9c318b436451e87479f7463f964f Mon Sep 17 00:00:00 2001 From: "D. V. Wiebe" Date: Fri, 10 Jul 2026 15:10:50 -0700 Subject: [PATCH 05/11] test: recover test_save_to_state --- old_tests/test_save_to_state.py | 92 --------------------------------- tests/test_save_to_state.py | 81 +++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 92 deletions(-) delete mode 100644 old_tests/test_save_to_state.py create mode 100644 tests/test_save_to_state.py diff --git a/old_tests/test_save_to_state.py b/old_tests/test_save_to_state.py deleted file mode 100644 index 7a4b7542..00000000 --- a/old_tests/test_save_to_state.py +++ /dev/null @@ -1,92 +0,0 @@ -"""Test endpoint config option `save_state` and `get_state`.""" - -import pytest -from coco.test import coco_runner - -SAVE_ENDPT_NAME = "save" -GET_ENDPT_NAME1 = "get1" -GET_ENDPT_NAME2 = "get2" - -CONFIG = {"log_level": "INFO", "groups": {"no_group": ["no_host", "doesnt_exist"]}} -INT_VAL = 5 -INT_VAL_NAME = "val" -STATE_PATH = "test_state" -ENDPOINTS = { - SAVE_ENDPT_NAME: { - "call": {"forward": None}, - "save_state": [STATE_PATH + "/1", STATE_PATH + "/2"], - "values": {INT_VAL_NAME: "int"}, - }, - GET_ENDPT_NAME1: {"call": {"forward": None}, "get_state": STATE_PATH + "/1"}, - GET_ENDPT_NAME2: {"call": {"forward": None}, "get_state": STATE_PATH + "/2"}, -} - - -@pytest.fixture -def runner(): - """Create a coco runner that doesn't reset on exit.""" - with coco_runner.Runner(CONFIG, ENDPOINTS, reset_on_shutdown=False) as runner: - yield runner - - -@pytest.fixture -def reset_runner(): - """Create a coco runner that resets the state on start.""" - with coco_runner.Runner(CONFIG, ENDPOINTS, reset_on_start=True) as runner: - yield runner - - -def test_save_state(runner): - """Test get/save_state.""" - # State should be empty now - response = runner.client(GET_ENDPT_NAME1) - assert "state" in response - assert STATE_PATH in response["state"] - assert "1" in response["state"][STATE_PATH] - assert response["state"][STATE_PATH]["1"] == {} - response = runner.client(GET_ENDPT_NAME2) - assert "state" in response - assert STATE_PATH in response["state"] - assert "2" in response["state"][STATE_PATH] - assert response["state"][STATE_PATH]["2"] == {} - - # Set state to INT_VAL - runner.client(SAVE_ENDPT_NAME, [str(INT_VAL)]) - response = runner.client(GET_ENDPT_NAME1) - assert "state" in response - assert STATE_PATH in response["state"] - assert "1" in response["state"][STATE_PATH] - assert response["state"][STATE_PATH]["1"] == {INT_VAL_NAME: INT_VAL} - response = runner.client(GET_ENDPT_NAME2) - assert "state" in response - assert STATE_PATH in response["state"] - assert "2" in response["state"][STATE_PATH] - assert response["state"][STATE_PATH]["2"] == {INT_VAL_NAME: INT_VAL} - - -def test_no_reset(runner): - # runner is set to not reset on shutdown, so we should still have the state - response = runner.client(GET_ENDPT_NAME1) - assert "state" in response - assert STATE_PATH in response["state"] - assert "1" in response["state"][STATE_PATH] - assert response["state"][STATE_PATH]["1"] == {INT_VAL_NAME: INT_VAL} - response = runner.client(GET_ENDPT_NAME2) - assert "state" in response - assert STATE_PATH in response["state"] - assert "2" in response["state"][STATE_PATH] - assert response["state"][STATE_PATH]["2"] == {INT_VAL_NAME: INT_VAL} - - -def test_reset(reset_runner): - # State should be empty again because reset_runner will flush it on start - response = reset_runner.client(GET_ENDPT_NAME1) - assert "state" in response - assert STATE_PATH in response["state"] - assert "1" in response["state"][STATE_PATH] - assert response["state"][STATE_PATH]["1"] == {} - response = reset_runner.client(GET_ENDPT_NAME2) - assert "state" in response - assert STATE_PATH in response["state"] - assert "2" in response["state"][STATE_PATH] - assert response["state"][STATE_PATH]["2"] == {} diff --git a/tests/test_save_to_state.py b/tests/test_save_to_state.py new file mode 100644 index 00000000..c2d854e0 --- /dev/null +++ b/tests/test_save_to_state.py @@ -0,0 +1,81 @@ +"""Test endpoint config option `save_state` and `get_state`.""" + +import pytest + + +@pytest.fixture +def runner(coco_runner): + """Create a coco runner with some endpoints.""" + + coco_runner.add_endpoint( + "save", + { + "call": {"forward": None}, + "save_state": ["test_state/1", "test_state/2"], + "values": {"val": "int"}, + }, + ) + coco_runner.add_endpoint( + "get1", {"call": {"forward": None}, "get_state": "test_state/1"} + ) + coco_runner.add_endpoint( + "get2", {"call": {"forward": None}, "get_state": "test_state/2"} + ) + + return coco_runner + + +def test_save_state(runner): + """Test get/save_state.""" + # State starts off empty + + response = runner.client("get1", decode=True) + assert "state" in response + assert "test_state" in response["state"] + assert "1" in response["state"]["test_state"] + assert response["state"]["test_state"]["1"] == {} + response = runner.client("get2", decode=True) + assert "state" in response + assert "test_state" in response["state"] + assert "2" in response["state"]["test_state"] + assert response["state"]["test_state"]["2"] == {} + + runner.client("save", "--val=5") + response = runner.client("get1", decode=True) + assert "state" in response + assert "test_state" in response["state"] + assert "1" in response["state"]["test_state"] + assert response["state"]["test_state"]["1"] == {"val": 5} + response = runner.client("get2", decode=True) + assert "state" in response + assert "test_state" in response["state"] + assert "2" in response["state"]["test_state"] + assert response["state"]["test_state"]["2"] == {"val": 5} + + +def test_no_reset(runner): + """Check state with it initially set.""" + runner.set_state({"test_state": {"1": {"val": 5}, "2": {"val": 5}}}) + response = runner.client("get1", decode=True) + assert "state" in response + assert "test_state" in response["state"] + assert "1" in response["state"]["test_state"] + assert response["state"]["test_state"]["1"] == {"val": 5} + response = runner.client("get2", decode=True) + assert "state" in response + assert "test_state" in response["state"] + assert "2" in response["state"]["test_state"] + assert response["state"]["test_state"]["2"] == {"val": 5} + + +def test_reset(runner): + response = runner.client("get1", decode=True) + assert "state" in response + assert "test_state" in response["state"] + assert "1" in response["state"]["test_state"] + assert response["state"]["test_state"]["1"] == {} + response = runner.client("get2", decode=True) + assert "state" in response + assert "test_state" in response["state"] + assert "2" in response["state"]["test_state"] + assert response["state"]["test_state"]["2"] == {} From 3ad40ce25c06be9f93a0615123790f57964a9374 Mon Sep 17 00:00:00 2001 From: "D. V. Wiebe" Date: Fri, 10 Jul 2026 15:37:45 -0700 Subject: [PATCH 06/11] test: recover test_scheduler --- old_tests/test_scheduler.py | 99 ------------------------------------- tests/test_scheduler.py | 86 ++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 99 deletions(-) delete mode 100644 old_tests/test_scheduler.py create mode 100644 tests/test_scheduler.py diff --git a/old_tests/test_scheduler.py b/old_tests/test_scheduler.py deleted file mode 100644 index ce93fee8..00000000 --- a/old_tests/test_scheduler.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Test endpoint scheduler.""" - -import json -import tempfile -import time - -import pytest -from coco.test import coco_runner, endpoint_farm - -CONFIG = {"log_level": "DEBUG"} -STATE_PATH = "test/success" -STATE_PATH_FAIL_TYPE = "test/fail_type" -STATE_PATH_FAIL_VAL = "test/fail_val" -PERIOD = 1.5 -ENDPOINTS = { - "scheduled": {"group": "test", "schedule": {"period": PERIOD}}, - "scheduled-check-type": { - "group": "test", - "schedule": { - "period": PERIOD, - "require_state": {"path": STATE_PATH, "type": "bool"}, - }, - }, - "scheduled-check-val": { - "group": "test", - "schedule": { - "period": PERIOD, - "require_state": {"path": STATE_PATH, "type": "bool", "value": True}, - }, - }, - "scheduled-fail-type": { - "group": "test", - "schedule": { - "period": PERIOD, - "require_state": {"path": STATE_PATH_FAIL_TYPE, "type": "bool"}, - }, - }, - "scheduled-fail-val": { - "group": "test", - "schedule": { - "period": PERIOD, - "require_state": { - "path": STATE_PATH_FAIL_VAL, - "type": "bool", - "value": True, - }, - }, - }, -} - - -def callback(data): - """Reply with the incoming json request.""" - return data - - -CALLBACKS = dict.fromkeys(ENDPOINTS, callback) -STATEFILE = tempfile.NamedTemporaryFile("w") -N_HOSTS = 2 - - -@pytest.fixture -def farm(): - """Create an endpoint test farm.""" - return endpoint_farm.Farm(N_HOSTS, CALLBACKS) - - -@pytest.fixture -def runner(farm): - """Create a coco runner.""" - CONFIG["groups"] = {"test": farm.hosts} - state = { - STATE_PATH.split("/")[1]: True, - STATE_PATH_FAIL_TYPE.split("/")[1]: "not_a_bool", - STATE_PATH_FAIL_VAL.split("/")[1]: False, - } - json.dump(state, STATEFILE) - STATEFILE.flush() - CONFIG["load_state"] = {STATE_PATH.split("/")[0]: STATEFILE.name} - print(STATEFILE.name) - with coco_runner.Runner(CONFIG, ENDPOINTS, reset_on_start=True) as runner: - yield runner - runner.stop_coco() - - -def test_sched(farm, runner): - """Test if scheduled endpoints are called when they should be.""" - start_t = time.time() - # Let three periods pass - time.sleep(3 * PERIOD + 1.4) - counters = farm.counters() - end_t = time.time() - num_sched = (end_t - start_t) // PERIOD - for p in farm.ports: - assert counters[p]["scheduled"] == num_sched - assert counters[p]["scheduled-check-type"] == num_sched - assert counters[p]["scheduled-check-val"] == num_sched - assert "scheduled-fail-type" not in counters[p] - assert "scheduled-fail-val" not in counters[p] diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py new file mode 100644 index 00000000..9b853d3c --- /dev/null +++ b/tests/test_scheduler.py @@ -0,0 +1,86 @@ +"""Test endpoint scheduler.""" + +import time + +import yaml + + +def test_sched(coco_runner): + """Test if scheduled endpoints are called when they should be.""" + PERIOD = 0.5 + + coco_runner.add_targets("test", 2, callback=lambda path, data: data) + coco_runner.add_endpoint( + "scheduled", {"group": "test", "schedule": {"period": PERIOD}} + ) + coco_runner.add_endpoint( + "scheduled-check-type", + { + "group": "test", + "schedule": { + "period": PERIOD, + "require_state": {"path": "test/success", "type": "bool"}, + }, + }, + ) + coco_runner.add_endpoint( + "scheduled-check-val", + { + "group": "test", + "schedule": { + "period": PERIOD, + "require_state": { + "path": "test/success", + "type": "bool", + "value": True, + }, + }, + }, + ) + coco_runner.add_endpoint( + "scheduled-fail-type", + { + "group": "test", + "schedule": { + "period": PERIOD, + "require_state": {"path": "test/fail_type", "type": "bool"}, + }, + }, + ) + coco_runner.add_endpoint( + "scheduled-fail-val", + { + "group": "test", + "schedule": { + "period": PERIOD, + "require_state": { + "path": "test/fail_val", + "type": "bool", + "value": True, + }, + }, + }, + ) + + # This is not a state file, but we'll throw it into the storage_path, just because + # it's a convenient place to put it. + with open(coco_runner.storage_path / "TEST.yaml", "w") as f: + yaml.dump({"success": True, "fail_type": "not_a_bool", "fail_val": False}, f) + coco_runner.add_config( + load_state={"test": str(coco_runner.storage_path / "TEST.yaml")} + ) + + coco_runner.start_daemon() + # Let at least three periods pass + time.sleep(3.5 * PERIOD) + coco_runner.stop() + + for t in coco_runner.targets: + # We don't know this precisely, because more periods may + # happen during the coco_runner daemon set-up and teardown + assert t.hit_count("/scheduled") >= 3 + num_sched = t.hit_count("/scheduled") + assert t.hit_count("/scheduled-check-type") == num_sched + assert t.hit_count("/scheduled-check-val") == num_sched + assert t.hit_count("/scheduled-fail-type") == 0 + assert t.hit_count("/scheduled-fail-val") == 0 From ee5f9df2fffd8241f3eff90232727fb52cec556c Mon Sep 17 00:00:00 2001 From: "D. V. Wiebe" Date: Fri, 10 Jul 2026 16:07:34 -0700 Subject: [PATCH 07/11] test: recover test_set_state --- old_tests/test_set_state.py | 218 ----------------------------------- tests/test_set_state.py | 224 ++++++++++++++++++++++++++++++++++++ 2 files changed, 224 insertions(+), 218 deletions(-) delete mode 100644 old_tests/test_set_state.py create mode 100644 tests/test_set_state.py diff --git a/old_tests/test_set_state.py b/old_tests/test_set_state.py deleted file mode 100644 index 787ef97f..00000000 --- a/old_tests/test_set_state.py +++ /dev/null @@ -1,218 +0,0 @@ -"""Test endpoint config option `set_state` and `get_state`.""" - -import pytest -from coco.test import coco_runner, endpoint_farm - -from coco.util import hash_dict - -SET_ENDPT_NAME = "set" -SET_ENDPT_NAME_INT = "set_int" -SET_ENDPT_NAME_DICT = "set_dict" -GET_ENDPT_NAME = "get" -CHECK_HASH_ENDPT_NAME = "check_hash" -CHECK_HASH_ENDPT_NAME2 = "check_hash2" -CHECK_STATE_ENDPT_NAME = "check_state" -CHECK_STATE_ENDPT_NAME2 = "check_state2" -HASH_ENDPT_NAME = "hash" -EXCLUDED_STATE_PATH = "excluded/from/reset" -CONFIG = { - "log_level": "DEBUG", - "groups": {"no_group": ["no_host", "doesnt_exist"]}, - "exclude_from_reset": [EXCLUDED_STATE_PATH], -} -INT_VAL = 5 -STATE_PATH = "test_state" -ENDPOINTS = { - "getall": {"call": {"forward": None}, "get_state": "/"}, - SET_ENDPT_NAME: {"call": {"forward": None}, "set_state": {STATE_PATH: True}}, - SET_ENDPT_NAME_INT: {"call": {"forward": None}, "set_state": {STATE_PATH: INT_VAL}}, - SET_ENDPT_NAME_DICT: { - "call": {"forward": None}, - "set_state": {STATE_PATH: {"s": {"n": {"a": "fu"}}}}, - }, - "set_excluded_state": { - "call": {"forward": None}, - "set_state": {EXCLUDED_STATE_PATH: INT_VAL}, - }, - GET_ENDPT_NAME: {"call": {"forward": None}, "get_state": STATE_PATH}, - "get_excluded_state": {"call": {"forward": None}, "get_state": EXCLUDED_STATE_PATH}, - CHECK_HASH_ENDPT_NAME: { - "group": "test", - "values": {"data": "str"}, - "call": { - "forward": {"name": HASH_ENDPT_NAME, "reply": {"state_hash": {"data": "/"}}} - }, - }, - CHECK_HASH_ENDPT_NAME2: { - "group": "test", - "values": {"data": "str"}, - "call": { - "forward": { - "name": HASH_ENDPT_NAME, - "reply": {"state_hash": {"data": "test_state/s/n"}}, - } - }, - }, - CHECK_STATE_ENDPT_NAME: { - "group": "test", - "values": {"data": "str"}, - "call": { - "forward": { - "name": HASH_ENDPT_NAME, - "reply": {"state": {"data": "test_state/s/n/a"}}, - } - }, - }, - CHECK_STATE_ENDPT_NAME2: { - "group": "test", - "values": {"n": "dict"}, - "call": { - "forward": {"name": HASH_ENDPT_NAME, "reply": {"state": "test_state/s/"}} - }, - }, -} - - -def callback(data): - """Reply with the incoming json request.""" - return data - - -N_HOSTS = 1 -CALLBACKS = {HASH_ENDPT_NAME: callback} - - -@pytest.fixture -def farm(): - """Create an endpoint test farm.""" - return endpoint_farm.Farm(N_HOSTS, CALLBACKS) - - -@pytest.fixture -def runner(farm): - """Create a coco runner.""" - CONFIG["groups"] = {"test": farm.hosts} - with coco_runner.Runner(CONFIG, ENDPOINTS) as runner: - yield runner - - -def test_get_state(runner): - """Test get/set_state.""" - - # Set state to True - runner.client(SET_ENDPT_NAME) - - # Get state and compare - response = runner.client(GET_ENDPT_NAME) - assert "state" in response - assert STATE_PATH in response["state"] - assert response["state"][STATE_PATH] is True - - # Set state to INT - runner.client(SET_ENDPT_NAME_INT) - - # Get state and compare - response = runner.client(GET_ENDPT_NAME) - assert "state" in response - assert STATE_PATH in response["state"] - assert response["state"][STATE_PATH] == 5 - - # Test passing check against state hash - runner.client(CHECK_HASH_ENDPT_NAME, [str(hash_dict({"test_state": 5}))]) - assert "failed_checks" not in response - - # Test failing check against state hash - response = runner.client(CHECK_HASH_ENDPT_NAME, [str(hash_dict({"foo": 5}))]) - assert "failed_checks" in response - assert HASH_ENDPT_NAME in response["failed_checks"] - for r in response["failed_checks"][HASH_ENDPT_NAME].values(): - assert r == {"reply": {"mismatch_with_state_hash": ["data"]}} - - response = runner.client(CHECK_HASH_ENDPT_NAME, [str(hash_dict({"test_state": 4}))]) - assert "failed_checks" in response - assert HASH_ENDPT_NAME in response["failed_checks"] - for r in response["failed_checks"][HASH_ENDPT_NAME].values(): - assert r == {"reply": {"mismatch_with_state_hash": ["data"]}} - - # The same for a part of the state: - runner.client(SET_ENDPT_NAME_DICT) - # Test passing check against partly state hash - response = runner.client(CHECK_HASH_ENDPT_NAME2, [str(hash_dict({"a": "fu"}))]) - assert "failed_checks" not in response - - # Test failing check against partly state hash - response = runner.client(CHECK_HASH_ENDPT_NAME2, [str(hash_dict({"a": "foo"}))]) - assert "failed_checks" in response - assert HASH_ENDPT_NAME in response["failed_checks"] - for r in response["failed_checks"][HASH_ENDPT_NAME].values(): - assert r == {"reply": {"mismatch_with_state_hash": ["data"]}} - - # Test checks against state - # Test passing check against part of state - response = runner.client(CHECK_STATE_ENDPT_NAME, ["fu"]) - assert "failed_checks" not in response - - # Test failing check against part of state - response = runner.client(CHECK_STATE_ENDPT_NAME, ["f00"]) - assert "failed_checks" in response - assert HASH_ENDPT_NAME in response["failed_checks"] - for r in response["failed_checks"][HASH_ENDPT_NAME].values(): - assert r == {"reply": {"mismatch_with_state": ["data"]}} - response = runner.client(CHECK_STATE_ENDPT_NAME, [""]) - assert "failed_checks" in response - assert HASH_ENDPT_NAME in response["failed_checks"] - for r in response["failed_checks"][HASH_ENDPT_NAME].values(): - assert r == {"reply": {"mismatch_with_state": ["data"]}} - - # Test passing check against state - response = runner.client(CHECK_STATE_ENDPT_NAME2, ['{"a": "fu"}']) - assert "failed_checks" not in response - - # Test failing check against state - response = runner.client(CHECK_STATE_ENDPT_NAME2, ['{"n": {"a": 0}}']) - assert "failed_checks" in response - assert HASH_ENDPT_NAME in response["failed_checks"] - for r in response["failed_checks"][HASH_ENDPT_NAME].values(): - assert r == {"reply": {"mismatch_with_state": ["all"]}} - response = runner.client(CHECK_STATE_ENDPT_NAME2, ['{"aa": "fu"}']) - assert "failed_checks" in response - assert HASH_ENDPT_NAME in response["failed_checks"] - for r in response["failed_checks"][HASH_ENDPT_NAME].values(): - assert r == {"reply": {"mismatch_with_state": ["all"]}} - response = runner.client(CHECK_STATE_ENDPT_NAME2, ["{}"]) - assert "failed_checks" in response - assert HASH_ENDPT_NAME in response["failed_checks"] - for r in response["failed_checks"][HASH_ENDPT_NAME].values(): - assert r == {"reply": {"mismatch_with_state": ["all"]}} - - -def test_reset_state(runner): - runner.client(SET_ENDPT_NAME_INT) - response = runner.client(GET_ENDPT_NAME) - assert "state" in response - assert STATE_PATH in response["state"] - assert response["state"][STATE_PATH] == INT_VAL - - runner.client("set_excluded_state") - response = runner.client("get_excluded_state") - assert "state" in response - path = EXCLUDED_STATE_PATH.split("/") - r = response["state"] - for p in path: - assert p in r - r = r[p] - assert r == INT_VAL - - runner.client("reset-state") - response = runner.client(GET_ENDPT_NAME) - assert "status_code" in response - assert response["status_code"] == 500 # path not found - - response = runner.client("get_excluded_state") - assert "state" in response - path = EXCLUDED_STATE_PATH.split("/") - r = response["state"] - for p in path: - assert p in r - r = r[p] - assert r == INT_VAL diff --git a/tests/test_set_state.py b/tests/test_set_state.py new file mode 100644 index 00000000..4f03537e --- /dev/null +++ b/tests/test_set_state.py @@ -0,0 +1,224 @@ +"""Test endpoint config option `set_state` and `get_state`.""" + +import pytest + +from coco.util import hash_dict + + +@pytest.fixture +def runner(coco_runner): + """A configured coco_runner.""" + + coco_runner.add_targets("test", 1, callback=lambda path, data: data) + coco_runner.add_config(exclude_from_reset=["excluded/from/reset"]) + coco_runner.add_endpoint("getall", {"call": {"forward": None}, "get_state": "/"}) + coco_runner.add_endpoint( + "set", {"call": {"forward": None}, "set_state": {"test_state": True}} + ) + coco_runner.add_endpoint( + "set_int", {"call": {"forward": None}, "set_state": {"test_state": 5}} + ) + coco_runner.add_endpoint( + "set_dict", + { + "call": {"forward": None}, + "set_state": {"test_state": {"s": {"n": {"a": "fu"}}}}, + }, + ) + coco_runner.add_endpoint( + "set_excluded_state", + { + "call": {"forward": None}, + "set_state": {"excluded/from/reset": 5}, + }, + ) + coco_runner.add_endpoint( + "get", {"call": {"forward": None}, "get_state": "test_state"} + ) + coco_runner.add_endpoint( + "get_excluded_state", + {"call": {"forward": None}, "get_state": "excluded/from/reset"}, + ) + coco_runner.add_endpoint( + "check_hash", + { + "group": "test", + "values": {"data": "str"}, + "call": { + "forward": {"name": "hash", "reply": {"state_hash": {"data": "/"}}} + }, + }, + ) + coco_runner.add_endpoint( + "check_hash2", + { + "group": "test", + "values": {"data": "str"}, + "call": { + "forward": { + "name": "hash", + "reply": {"state_hash": {"data": "test_state/s/n"}}, + } + }, + }, + ) + coco_runner.add_endpoint( + "check_state", + { + "group": "test", + "values": {"data": "str"}, + "call": { + "forward": { + "name": "hash", + "reply": {"state": {"data": "test_state/s/n/a"}}, + } + }, + }, + ) + coco_runner.add_endpoint( + "check_state2", + { + "group": "test", + "values": {"n": "dict"}, + "call": {"forward": {"name": "hash", "reply": {"state": "test_state/s/"}}}, + }, + ) + + return coco_runner + + +def test_get_state(runner): + """Test get/set_state.""" + + # Set state to True + runner.client("set") + + # Get state and compare + response = runner.client("get", decode=True) + assert "state" in response + assert "test_state" in response["state"] + assert response["state"]["test_state"] is True + + # Set state to INT + runner.client("set_int") + + # Get state and compare + response = runner.client("get", decode=True) + assert "state" in response + assert "test_state" in response["state"] + assert response["state"]["test_state"] == 5 + + # Test passing check against state hash + response = runner.client( + "-r", "FULL", "check_hash", "--data", hash_dict({"test_state": 5}), decode=True + ) + assert "failed_checks" not in response + + # Test failing check against state hash + response = runner.client( + "-r", "FULL", "check_hash", "--data", hash_dict({"foo": 5}), decode=True + ) + assert "failed_checks" in response + assert "hash" in response["failed_checks"] + for r in response["failed_checks"]["hash"].values(): + assert r == {"reply": {"mismatch_with_state_hash": ["data"]}} + + response = runner.client( + "-r", "FULL", "check_hash", "--data", hash_dict({"test_state": 4}), decode=True + ) + assert "failed_checks" in response + assert "hash" in response["failed_checks"] + for r in response["failed_checks"]["hash"].values(): + assert r == {"reply": {"mismatch_with_state_hash": ["data"]}} + + # The same for a part of the state: + runner.client("set_dict") + # Test passing check against partly state hash + response = runner.client( + "check_hash2", "--data", hash_dict({"a": "fu"}), decode=True + ) + assert "failed_checks" not in response + + # Test failing check against partly state hash + response = runner.client( + "-r", "FULL", "check_hash2", "--data", hash_dict({"a": "foo"}), decode=True + ) + assert "failed_checks" in response + assert "hash" in response["failed_checks"] + for r in response["failed_checks"]["hash"].values(): + assert r == {"reply": {"mismatch_with_state_hash": ["data"]}} + + # Test checks against state + # Test passing check against part of state + response = runner.client("check_state", "--data=fu", decode=True) + assert "failed_checks" not in response + + # Test failing check against part of state + response = runner.client("-r", "FULL", "check_state", "--data=f00", decode=True) + assert "failed_checks" in response + assert "hash" in response["failed_checks"] + for r in response["failed_checks"]["hash"].values(): + assert r == {"reply": {"mismatch_with_state": ["data"]}} + + response = runner.client("-r", "FULL", "check_state", "--data=", decode=True) + assert "failed_checks" in response + assert "hash" in response["failed_checks"] + for r in response["failed_checks"]["hash"].values(): + assert r == {"reply": {"mismatch_with_state": ["data"]}} + + # Test passing check against state + response = runner.client("check_state2", "-n", '{"a": "fu"}', decode=True) + assert "failed_checks" not in response + + # Test failing check against state + response = runner.client( + "-r", "FULL", "check_state2", "-n", '{"n": {"a": 0}}', decode=True + ) + assert "failed_checks" in response + assert "hash" in response["failed_checks"] + for r in response["failed_checks"]["hash"].values(): + assert r == {"reply": {"mismatch_with_state": ["all"]}} + + response = runner.client( + "-r", "FULL", "check_state2", "-n", '{"aa": "fu"}', decode=True + ) + assert "failed_checks" in response + assert "hash" in response["failed_checks"] + for r in response["failed_checks"]["hash"].values(): + assert r == {"reply": {"mismatch_with_state": ["all"]}} + + response = runner.client("-r", "FULL", "check_state2", "-n", "{}", decode=True) + assert "failed_checks" in response + assert "hash" in response["failed_checks"] + for r in response["failed_checks"]["hash"].values(): + assert r == {"reply": {"mismatch_with_state": ["all"]}} + + +def test_reset_state(runner): + runner.client("set_int") + response = runner.client("get", decode=True) + assert "state" in response + assert "test_state" in response["state"] + assert response["state"]["test_state"] == 5 + + runner.client("set_excluded_state") + response = runner.client("get_excluded_state", decode=True) + assert "state" in response + r = response["state"] + for p in ("excluded", "from", "reset"): + assert p in r + r = r[p] + assert r == 5 + + runner.client("reset-state") + response = runner.client("get", decode=True) + assert "status_code" in response + assert response["status_code"] == 500 # path not found + + response = runner.client("get_excluded_state", decode=True) + assert "state" in response + r = response["state"] + for p in ("excluded", "from", "reset"): + assert p in r + r = r[p] + assert r == 5 From 226a7ebab8a63dfd350c90bfe8dea6dc3cf75814 Mon Sep 17 00:00:00 2001 From: "D. V. Wiebe" Date: Fri, 10 Jul 2026 16:24:07 -0700 Subject: [PATCH 08/11] test: recover test_timeout Also: * handle broken pipe in rest_server fixture --- old_tests/test_timeout.py | 62 --------------------------------------- tests/rest_server.py | 5 +++- tests/test_timeout.py | 36 +++++++++++++++++++++++ 3 files changed, 40 insertions(+), 63 deletions(-) delete mode 100644 old_tests/test_timeout.py create mode 100644 tests/test_timeout.py diff --git a/old_tests/test_timeout.py b/old_tests/test_timeout.py deleted file mode 100644 index cbf40d85..00000000 --- a/old_tests/test_timeout.py +++ /dev/null @@ -1,62 +0,0 @@ -"""Test basic endpoint call forwarding with timeout.""" - -import time - -import pytest -from coco.test import coco_runner, endpoint_farm - -ENDPT_NAME = "test" -PORT = 12055 -N_CALLS = 2 -CONFIG = { - "log_level": "INFO", - "port": PORT, - "timeout": "10s", - "debug_connections": True, -} -ENDPOINTS = { - ENDPT_NAME: { - "call": {"forward": {"name": ENDPT_NAME, "timeout": "1s"}}, - "group": "test", - "values": {"foo": "int", "bar": "str"}, - } -} - - -def callback(data): - """Reply with the incoming json request.""" - time.sleep(2) - return data - - -N_HOSTS = 2 -CALLBACKS = {ENDPT_NAME: callback} - - -@pytest.fixture(scope="module") -def farm(): - """Create a coco runner.""" - return endpoint_farm.Farm(N_HOSTS, CALLBACKS) - - -@pytest.fixture(scope="module") -def runner(farm): - """Create an endpoint test farm.""" - CONFIG["groups"] = {"test": farm.hosts} - with coco_runner.Runner(CONFIG, ENDPOINTS) as runner: - yield runner - - -def test_timeout_for_specific_forward(farm, runner): - response = runner.client(ENDPT_NAME, ["0", "1337"]) - - for p in farm.ports: - assert farm.counters()[p][ENDPT_NAME] == 1 - assert ENDPT_NAME in response - for h in farm.hosts: - assert h in response[ENDPT_NAME] - assert "status" in response[ENDPT_NAME][h] - assert "reply" in response[ENDPT_NAME][h] - - assert response[ENDPT_NAME][h]["status"] == 0 - assert response[ENDPT_NAME][h]["reply"] == "Timeout" diff --git a/tests/rest_server.py b/tests/rest_server.py index 2b285887..ef0e0952 100644 --- a/tests/rest_server.py +++ b/tests/rest_server.py @@ -73,7 +73,10 @@ def send_json(self, body, response): self.end_headers() # Response goes in the body - self.wfile.write(response.encode()) + try: + self.wfile.write(response.encode()) + except BrokenPipeError: + pass return def handle_route(self): diff --git a/tests/test_timeout.py b/tests/test_timeout.py new file mode 100644 index 00000000..37dba5b6 --- /dev/null +++ b/tests/test_timeout.py @@ -0,0 +1,36 @@ +"""Test basic endpoint call forwarding with timeout.""" + +import time + + +def callback(path, data): + """Reply with the incoming json request.""" + time.sleep(2) + return data + + +def test_timeout_for_specific_forward(coco_runner): + """Test endpoint timeout""" + + coco_runner.add_targets("test", 2, callback=callback) + coco_runner.add_endpoint( + "test", + { + "call": {"forward": {"name": "test", "timeout": "1s"}}, + "group": "test", + "values": {"foo": "int", "bar": "str"}, + }, + ) + + response = coco_runner.client( + "-r", "FULL", "test", "--foo=0", "--bar=1337", decode=True + ) + + assert "test" in response + for t in coco_runner.targets: + h = f"http://127.0.0.1:{t.port}/" + assert h in response["test"] + assert "status" in response["test"][h] + assert "reply" in response["test"][h] + assert response["test"][h]["status"] == 0 + assert response["test"][h]["reply"] == "Timeout" From 5ac2dd81454115632d96845163a61ec9854e9078 Mon Sep 17 00:00:00 2001 From: "D. V. Wiebe" Date: Fri, 10 Jul 2026 16:27:59 -0700 Subject: [PATCH 09/11] test: recover test_timeout_global --- old_tests/test_timeout_global.py | 51 -------------------------------- tests/test_timeout_global.py | 37 +++++++++++++++++++++++ 2 files changed, 37 insertions(+), 51 deletions(-) delete mode 100644 old_tests/test_timeout_global.py create mode 100644 tests/test_timeout_global.py diff --git a/old_tests/test_timeout_global.py b/old_tests/test_timeout_global.py deleted file mode 100644 index 3983a98e..00000000 --- a/old_tests/test_timeout_global.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Test basic endpoint call forwarding with timeout.""" - -import time - -import pytest -from coco.test import coco_runner, endpoint_farm - -ENDPT_NAME = "test" -PORT = 12055 -CONFIG = {"log_level": "INFO", "port": PORT, "timeout": "1s"} -ENDPOINTS = {ENDPT_NAME: {"group": "test", "values": {"foo": "int", "bar": "str"}}} -N_CALLS = 2 - - -def callback(data): - """Reply with the incoming json request.""" - time.sleep(2) - return data - - -N_HOSTS = 2 -CALLBACKS = {ENDPT_NAME: callback} - - -@pytest.fixture(scope="module") -def farm(): - """Create a coco runner.""" - return endpoint_farm.Farm(N_HOSTS, CALLBACKS) - - -@pytest.fixture(scope="module") -def runner(farm): - """Create an endpoint test farm.""" - CONFIG["groups"] = {"test": farm.hosts} - with coco_runner.Runner(CONFIG, ENDPOINTS) as runner: - yield runner - - -def test_timeout_global(farm, runner): - response = runner.client(ENDPT_NAME, ["0", "1337"]) - - for p in farm.ports: - assert farm.counters()[p][ENDPT_NAME] == 1 - assert ENDPT_NAME in response - for h in farm.hosts: - assert h in response[ENDPT_NAME] - assert "status" in response[ENDPT_NAME][h] - assert "reply" in response[ENDPT_NAME][h] - - assert response[ENDPT_NAME][h]["status"] == 0 - assert response[ENDPT_NAME][h]["reply"] == "Timeout" diff --git a/tests/test_timeout_global.py b/tests/test_timeout_global.py new file mode 100644 index 00000000..fffb4146 --- /dev/null +++ b/tests/test_timeout_global.py @@ -0,0 +1,37 @@ +"""Test basic endpoint call forwarding with timeout.""" + +import time + + +def callback(path, data): + """Reply with the incoming json request.""" + time.sleep(2) + return data + + +def test_timeout_global(coco_runner): + """Test global timeout""" + + coco_runner.add_config(timeout="1s") + coco_runner.add_targets("test", 2, callback=callback) + coco_runner.add_endpoint( + "test", + { + "call": {"forward": {"name": "test"}}, + "group": "test", + "values": {"foo": "int", "bar": "str"}, + }, + ) + + response = coco_runner.client( + "-r", "FULL", "test", "--foo=0", "--bar=1337", decode=True + ) + + assert "test" in response + for t in coco_runner.targets: + h = f"http://127.0.0.1:{t.port}/" + assert h in response["test"] + assert "status" in response["test"][h] + assert "reply" in response["test"][h] + assert response["test"][h]["status"] == 0 + assert response["test"][h]["reply"] == "Timeout" From 487c85f7ce44a92f03ec2538cf4e46856902ce9f Mon Sep 17 00:00:00 2001 From: "D. V. Wiebe" Date: Fri, 10 Jul 2026 16:52:43 -0700 Subject: [PATCH 10/11] test: recover test_wait Also: * Add all the local endpoint names to the list of all endpoints we check forwards against --- coco/core.py | 22 +++++++++- old_tests/test_wait.py | 94 ------------------------------------------ tests/test_wait.py | 65 +++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 96 deletions(-) delete mode 100644 old_tests/test_wait.py create mode 100644 tests/test_wait.py diff --git a/coco/core.py b/coco/core.py index fb5fb12a..e116ac08 100644 --- a/coco/core.py +++ b/coco/core.py @@ -107,8 +107,22 @@ def __init__( # Now that the state is loaded, validate all the endpoints defined by the config endpoints = [] groups = self.config["groups"] + all_endpoints = {endpoint["name"] for endpoint in self.config["endpoints"]} + # These are all the local endpoints + all_local_endpoints = { + "blocklist", + "update-blocklist", + "saved-states", + "reset-state", + "save-state", + "load-state", + "get-coco-config", + "wait", + } + all_endpoints |= all_local_endpoints + for endpoint in self.config["endpoints"]: endpoints.append( validate_endpoint(endpoint, groups, all_endpoints, self.state) @@ -134,7 +148,7 @@ def __init__( self._config_slack_loggers() self._load_endpoints() - self._local_endpoints() + self._local_endpoints(all_local_endpoints) self._check_endpoint_links() try: @@ -440,7 +454,7 @@ def _load_endpoints(self): ) self.forwarder.add_endpoint(name, self.endpoints[name]) - def _local_endpoints(self): + def _local_endpoints(self, all_local_endpoints): # Register any local endpoints endpoints = { @@ -454,6 +468,10 @@ def _local_endpoints(self): "wait": ("POST", wait.process_post), } + # Verify that everything's defined + if all_local_endpoints != set(endpoints): + raise RuntimeError("local endpoint mismatch") + for name, (type_, callable_) in endpoints.items(): self.endpoints[name] = LocalEndpoint(name, type_, callable_) self.forwarder.add_endpoint(name, self.endpoints[name]) diff --git a/old_tests/test_wait.py b/old_tests/test_wait.py deleted file mode 100644 index bc99a273..00000000 --- a/old_tests/test_wait.py +++ /dev/null @@ -1,94 +0,0 @@ -"""Test the internal WAIT endpoint.""" - -import time - -import pytest -from coco.test import coco_runner, endpoint_farm - -ENDPT_NAME = "test" -TS_ENDPT_NAME = "ts_endpt" -GET_TS_ENDPT_NAME = "get_ts_endpt" -TS_PATH = "timestamps/test" -CONFIG = {"log_level": "INFO"} -T_WAIT = 2 -T_HOW_SLOW_IS_COCO = 2.5 -ENDPOINTS = { - ENDPT_NAME: { - "group": "test", - "values": {"foo": "int", "bar": "str"}, - "call": {"coco": {"name": "wait", "request": {"duration": f"{T_WAIT}s"}}}, - }, - TS_ENDPT_NAME: {"group": "test", "timestamp": TS_PATH}, - GET_TS_ENDPT_NAME: {"group": "test", "get_state": TS_PATH}, -} -N_CALLS = 2 - - -def callback(data): - """Reply with the incoming json request.""" - return data - - -N_HOSTS = 2 -CALLBACKS = {ENDPT_NAME: callback} - - -@pytest.fixture -def farm(): - """Create a coco runner.""" - return endpoint_farm.Farm(N_HOSTS, CALLBACKS) - - -@pytest.fixture -def runner(farm): - """Create an endpoint test farm.""" - CONFIG["groups"] = {"test": farm.hosts} - with coco_runner.Runner(CONFIG, ENDPOINTS) as runner: - yield runner - - -def test_forward(farm, runner): - """Test if a request gets forwarded to an external endpoint.""" - request = {"foo": 0, "bar": "1337"} - t0 = time.time() - response = runner.client(ENDPT_NAME, ["0", "1337"]) - t1 = time.time() - assert t1 - t0 > T_WAIT - assert t1 - t0 < T_WAIT + T_HOW_SLOW_IS_COCO - - for p in farm.ports: - assert farm.counters()[p][ENDPT_NAME] == 1 - assert ENDPT_NAME in response - for h in farm.hosts: - assert h in response[ENDPT_NAME] - assert "status" in response[ENDPT_NAME][h] - assert "reply" in response[ENDPT_NAME][h] - - assert response[ENDPT_NAME][h]["status"] == 200 - assert response[ENDPT_NAME][h]["reply"] == request - - -def test_timestamp(farm, runner): - response = runner.client(TS_ENDPT_NAME) - for p in farm.ports: - assert farm.counters()[p][TS_ENDPT_NAME] == 1 - assert TS_ENDPT_NAME in response - for h in farm.hosts: - assert h in response[TS_ENDPT_NAME] - assert "status" in response[TS_ENDPT_NAME][h] - assert "reply" in response[TS_ENDPT_NAME][h] - - assert response[TS_ENDPT_NAME][h]["status"] == 200 - - response = runner.client(GET_TS_ENDPT_NAME) - for p in farm.ports: - assert farm.counters()[p][GET_TS_ENDPT_NAME] == 1 - assert GET_TS_ENDPT_NAME in response - assert "state" in response - assert "timestamps" in response["state"] - assert "test" in response["state"]["timestamps"] - timestamp = response["state"]["timestamps"]["test"] - assert isinstance(timestamp, float) - # This timestamps should be fresh. Test that it's between 0 and 10s old. - assert time.time() - timestamp > 0 - assert time.time() - timestamp < 10 diff --git a/tests/test_wait.py b/tests/test_wait.py new file mode 100644 index 00000000..236b3aeb --- /dev/null +++ b/tests/test_wait.py @@ -0,0 +1,65 @@ +"""Test the internal WAIT endpoint. Also test timestamps.""" + +import time + + +def test_wait(coco_runner): + """Test the wait endpoint.""" + + coco_runner.add_targets("test", 2) + coco_runner.add_endpoint( + "test", + { + "group": "test", + "values": {"foo": "int", "bar": "str"}, + "call": {"coco": {"name": "wait", "request": {"duration": "2s"}}}, + }, + ) + + t0 = time.monotonic() + response = coco_runner.client( + "-r", "FULL", "test", "--foo=0", "--bar=1337", decode=True + ) + t1 = time.monotonic() + assert t1 - t0 > 2 + assert t1 - t0 < 5 + + assert "test" in response + for t in coco_runner.targets: + assert t.hit_count("/test") == 1 + h = f"http://127.0.0.1:{t.port}/" + assert h in response["test"] + assert "status" in response["test"][h] + assert response["test"][h]["status"] == 200 + + +def test_timestamp(coco_runner): + """Test timestamps""" + + coco_runner.add_targets("test", 2, callback=lambda path, data: data) + coco_runner.add_endpoint( + "ts_endpt", {"group": "test", "timestamp": "timestamp/test"} + ) + coco_runner.add_endpoint( + "get_ts_endpt", {"group": "test", "get_state": "timestamp/test"} + ) + + response = coco_runner.client("ts_endpt", decode=True) + assert "ts_endpt" in response + for t in coco_runner.targets: + assert t.hit_count("/ts_endpt") == 1 + + response = coco_runner.client("-r", "FULL", "get_ts_endpt", decode=True) + for t in coco_runner.targets: + assert t.hit_count("/get_ts_endpt") == 1 + + assert "get_ts_endpt" in response + assert "state" in response + assert "timestamp" in response["state"] + assert "test" in response["state"]["timestamp"] + timestamp = response["state"]["timestamp"]["test"] + assert isinstance(timestamp, float) + + # This timestamps should be fresh. Test that it's between 0 and 10s old. + assert time.time() - timestamp > 0 + assert time.time() - timestamp < 10 From 48282b59a3979f1911afd1a23dffbe08fc56d852 Mon Sep 17 00:00:00 2001 From: "D. V. Wiebe" Date: Fri, 10 Jul 2026 16:59:07 -0700 Subject: [PATCH 11/11] test: delete unsalvalgeable stuff from old_tests This is the simulate-chime stuff and the test_saved_states.py whose tests have been duplicated --- old_tests/simulate-chime/Dockerfile | 37 - old_tests/simulate-chime/base.yaml | 27 - old_tests/simulate-chime/coco.conf | 43 - old_tests/simulate-chime/comet/Dockerfile | 31 - old_tests/simulate-chime/data/init_gain.hdf5 | Bin 14706 -> 0 bytes .../simulate-chime/data/update_gain.hdf5 | Bin 14762 -> 0 bytes old_tests/simulate-chime/docker-compose.yaml | 60 -- old_tests/simulate-chime/gps_server.json | 1 - old_tests/simulate-chime/gpu.yaml | 790 ------------------ old_tests/simulate-chime/recv.yaml | 306 ------- old_tests/simulate-chime/run_test.sh | 43 - old_tests/simulate-chime/test_client.py | 460 ---------- old_tests/test_saved_states.py | 106 --- 13 files changed, 1904 deletions(-) delete mode 100644 old_tests/simulate-chime/Dockerfile delete mode 100644 old_tests/simulate-chime/base.yaml delete mode 100644 old_tests/simulate-chime/coco.conf delete mode 100644 old_tests/simulate-chime/comet/Dockerfile delete mode 100644 old_tests/simulate-chime/data/init_gain.hdf5 delete mode 100644 old_tests/simulate-chime/data/update_gain.hdf5 delete mode 100644 old_tests/simulate-chime/docker-compose.yaml delete mode 100644 old_tests/simulate-chime/gps_server.json delete mode 100644 old_tests/simulate-chime/gpu.yaml delete mode 100644 old_tests/simulate-chime/recv.yaml delete mode 100755 old_tests/simulate-chime/run_test.sh delete mode 100644 old_tests/simulate-chime/test_client.py delete mode 100644 old_tests/test_saved_states.py diff --git a/old_tests/simulate-chime/Dockerfile b/old_tests/simulate-chime/Dockerfile deleted file mode 100644 index 7afca649..00000000 --- a/old_tests/simulate-chime/Dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -# Use an official Python runtime as a base image -FROM ubuntu:xenial - -## The maintainer name and email -MAINTAINER Rick Nitsche - -RUN set -xe - -# Install any needed packages specified in requirements.txt -RUN apt-get update -RUN apt-get install -y gcc -RUN apt-get install -y cmake -RUN apt-get install -y libhdf5-10 libhdf5-10-dbg libhdf5-dev h5utils -RUN apt-get install -y libboost-all-dev -RUN apt-get install -y python python-setuptools python-pip -RUN apt-get install -y libevent-dev -RUN apt-get install -y git -RUN apt-get install -y libssl-dev -RUN pip install pyyaml -RUN pip install yamllint - -# Minimize container size -RUN apt-get autoremove -y -RUN apt-get clean -y -RUN cd / -RUN rm -rf /tmp/build - -# Build kotekan -RUN mkdir -p /code -COPY ./kotekan /code/kotekan -COPY ./highfive /code/highfive -RUN cd /code/kotekan/build && cmake -DUSE_HDF5=ON -DHIGHFIVE_PATH=$PWD/../../highfive .. -RUN cd /code/kotekan/build && make -j 4 - -# Run kotekan when the container launches -WORKDIR /code/kotekan/build/kotekan/ -CMD ./kotekan -o \ No newline at end of file diff --git a/old_tests/simulate-chime/base.yaml b/old_tests/simulate-chime/base.yaml deleted file mode 100644 index b88dd9ce..00000000 --- a/old_tests/simulate-chime/base.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# Docker compose configuration options that will be shared between node instances -version: '2.3' # extends function removed in v3 -services: - gpu: - build: . - #volumes: - # - ../../../:/code - networks: - - receiver_net - ulimits: - memlock: 1000000000 - # should overwrite these in compose file - hostname: cn... # Does this work? - receiver: - build: . - volumes: - #- ../../../:/code - - ../data:/data - networks: - - receiver_net - ulimits: - memlock: 1000000000 - # should overwrite these in compose file - hostname: recv... -networks: - receiver_net: - driver: bridge diff --git a/old_tests/simulate-chime/coco.conf b/old_tests/simulate-chime/coco.conf deleted file mode 100644 index eb40a889..00000000 --- a/old_tests/simulate-chime/coco.conf +++ /dev/null @@ -1,43 +0,0 @@ -log_level: "DEBUG" -endpoint_dir: '../conf/endpoints' -host: localhost -port: 12055 -metrics_port: 12056 -n_workers: 2 -groups: - all: - - localhost:12100 - - localhost:12101 - - localhost:12102 - - localhost:12103 - - localhost:12104 - - localhost:12105 - - localhost:12106 - - localhost:12107 - - localhost:12108 - - localhost:12109 - - localhost:12049 - cluster: - - localhost:12100 - - localhost:12101 - - localhost:12102 - - localhost:12103 - - localhost:12104 - - localhost:12105 - - localhost:12106 - - localhost:12107 - - localhost:12108 - - localhost:12109 - receiver_nodes: - - localhost:12049 - gps_server: - - localhost:54321 -comet_broker: - enabled: True - host: localhost - port: 12050 -load_state: - cluster: "../tests/simulate-chime/gpu.yaml" - receiver: "../tests/simulate-chime/recv.yaml" -storage_path: "storage.json" -blocklist_path: "blocklist.json" diff --git a/old_tests/simulate-chime/comet/Dockerfile b/old_tests/simulate-chime/comet/Dockerfile deleted file mode 100644 index a64d334d..00000000 --- a/old_tests/simulate-chime/comet/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -# Use an official Python runtime as a base image -FROM ubuntu:xenial - -## The maintainer name and email -MAINTAINER Rick Nitsche - -RUN set -xe - -# Install any needed packages specified in requirements.txt -RUN apt-get update -RUN apt-get install -y software-properties-common -RUN add-apt-repository ppa:jonathonf/python-3.6 -RUN apt-get update -RUN apt-get install -y python3.6 python3.6-dev -RUN apt-get install -y git -RUN apt-get install -y curl -RUN apt-get install -y build-essential # autoconf libtool pkg-config -RUN curl https://bootstrap.pypa.io/get-pip.py | python3.6 - -RUN git clone https://github.com/chime-experiment/comet.git -RUN pip install -r /comet/requirements.txt -RUN pip install /comet - -# Minimize container size -RUN apt-get remove -y curl git -RUN apt-get autoremove -y -RUN apt-get clean -y -RUN rm -rf /tmp/build - -# Run comet when the container launches -CMD comet --debug 1 --recover 0 diff --git a/old_tests/simulate-chime/data/init_gain.hdf5 b/old_tests/simulate-chime/data/init_gain.hdf5 deleted file mode 100644 index e016f76e25d1d26d8e87c6726bea26fd103ee63e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14706 zcmeI2XIK=;+OTH^h9o&Bi2{P62r5|JjdYcm38HIQR}nCPpdw;Ij37Z2L|hdYPyrPX zb;+VAOqH&J3o3#b@F-x;VHM0TX?nxiKIg}GUGIpJ{3Bdu>W#BX|;x#Y3H-pwV4U@6a;$fJNJURj|IKwdk_Am_`9~^&Z~Fb zu}IW&;|je`q>QlpwmNIf{`6>R_MYx)v&+Bq2kB)ddwPu$3S^{>1ZFbQn!hil&wZR+ zPkmQ^U7LehcOm$>_q_DttHeF$|F3`kzk8+MJ)VCz)?K;dA8)hXAyMzrk=y<_-l^T= z9mv)FZ|v~>ocOCJf1L-UeqHYMNP4e9i$j-pKK_FFVe^)Dos)X!)^+gD#$_PzoF;ka zT~fKQ!xXih2*S_342izdW4M{PR$U)&N5b#7;@s+MF!vcE-+q3A#|C|4&GLqm`kgsg zX37ZS7d#BkcsC4<$li{XX&dyb=}b6POO||{atpr6YlY*O40tA3Ky;?6k?N$QZk5NQ z*sTEr(3nZBQ0>S~eCmE5q7vl{h3U`m;sCY8DY^K2hfVNpg%DjaE`V2-y~2mw zl@Md&hxgjNWud1$QJbMcTFWJppy4;F{JyE9uK?nrr(*P2NgmaUhvOg0=w?tmxAqO9z)&JDbUkrC!Rk=gl3H0 z2#r*g(D7O=;x*zZyJ+!u*as<~s%0&3|HX2cq;eTHHSWXvltrlGq7MAUJr`ce6O&bw z8ldCsXSg&ci`DfuN7Y+o&}fTI@bJ?*nCA8fOA{aaOqSf+aTFW$Im;eiIDotjON2*X zTcTxygmA=JM}!I$F$}Uq+iE_;&xZ~`=He%~Zv08O_r^UOX?GhhjTe(Q3v#f$lQC(! zy%}E!8;15CR>JfAKEiT20xwf(*kQ5`iHy-EPD=fVMP3p6<**ZZTe}8YhuV^YUpL~H z^9@MNh6L>6qmRa%(?+XG6D8!C7TN4H3bzzKmJA=Hjmk}opmmrT3KN?X<-1|u5xgI7f{Kbeuw8O3lx--5cP(_tv7rHY z_K#Y``ltrdyqW~3&-{^WNKf$Gz&vA;o26?wO23Lld5;n~YYZQOOCx=YL zKb836SIu%{iQaAOd9WBhc$1H}7(9c{O}ik7yN4eoY7r#|11vF8B0u_^hMOGwyS-C3 zCSmsbMqX-DLl3{yvX1_mXm+@eO!c#ZbBjJoPU|Y8EUghZAjKB_GSdvz9C*!*Ysi2? zob{oMJ3f^#j$L1EjOJA+5$~~w;Tcx{nWVjhyTL1i?+Xak%v(in;LQNvWj|y z+n05)Z>|m}=D|5Q(bR<~*0?}fxnaoOB@qXPTB9lYGvMwWvP5$D7Cb$;1uk*OfHEBd z;`K$1xHjE%`&A~6J$Bv>b^Ou-zo|51#Rp2{;Y(N8j$YtV39&e8&;_V&djzkD*bJ#~ zA^Q607);lGgU3-y=ybTA&i6xKQBx6#vP~}x4xO%=R zTBvSDqSTkcA<6nAXvNd21?jmsc*<*R6+8@o{Yeq632VhEKh?sjph9?HYF{)WuN}vw z)WYp$rBK~jn~Zwuk4r?$*@Gqd5gGt;8o}riKs=FoJo))U#CsO-?WQxYThk4u4XTsWB&rz6>5;@uF=>& z$dt%v8{$(R1*B-p7@T<{0Ed{%kt>G;VO8zQ@7EIUL$2YcQSp=Ft3 zB+OfnV=b=WSqTrYr@JvCwPPi^seMpJ{5lx8+7Q*B-GWumHNxV-O2|FE2HvlbC;P?f z=!$bBetgaVsfP@NE;T!FXrwxcOj09ZNziR;tP#4sdpS9LP$yR7EnF_b7h{?fc zx8Q)tJFvev1Nv_i5_Y#bnL0tt>LqMtEe!3@JdGCUxcoL2)GHEE@@QDo^b%j~h{lSR z=i!Xc`8fR7I9OmTM4?;rq0_#%*uqQ+)i;FT5vZA+YpO^LB9+OC16dMtL1I>>{G}**$!(u67eWK8B|%n8SYJ1M0+=Ak>Hv#R%q&phlePjb(dP9$JSza zkG=#omS*GXb0TzDR~csRI|y;Bm=vwJ4F3gQ;FtY=X7}csqQDO#^x)GP_(8T74lMb7 z9xz=cOCqZeI+_bV3jvlH-9!C4){R6c~{zo+wK5r{5d7jFC?`MooOq`6Tu_JkV4f%6gtMLhc6@h{z%j`c6&z>U@Vyh=j9a^ifIlRy+hW!pbX( zXq$62tYqYgYN9F%N?D6{18)}_QhgtF>GQ?7Y>{x2Tx{OI?hkkF1@(108X?OpHTXGc0 zngyQFXWDDL2W-M^Z_dHe+GE(%dMhj~5uki=9(31yhnKunMDI_A;epzh+4CP1$-wu0 zNJ&YGq|$g?)xa2a)Huxw*AJ6NlFPE_i+DUvS*?hE3jPT`yVn<~Pe{eTy=;J$gWp0e zA0<|K;a6O-DFctJxdm^R-^B@YlJH8mXRz8U3HCAGilgSrpa}Cg=<3< z+5r)AT(J^n)?S3lw|<`o_S?&n-!^9BQ*Vy3s-Cvwm*Xigi#0`M8c$h|puxy0L;?Re z-3(=q`~*L}OoRS{4!GQ^7}9BNc;(CMcyXl+x&MA24$(3qR(Dt7-wrw=y;lbKZSyC% zBklxTbSMKSjnE(tjcTOjgc%v1kj6&k47^CvazU1ZGFsLl2N2V%NR^_kH!{>sZ;b+4J zV>xR@G@-s3`-arQjtf7-*a$;(_vLRmd1@_WG)rNcfhO@0_+#}3P4fDqDw^-N1vGp~9@0W9)XuX^raAI^GMAeNn&zZRDqm?6>s@k0P#F)B;%RA>J zucm|e>~GKE7?&it|K%f0lC?U2X367IHS*+4h6Khea;nUS)sz0U1za2b*(0N@{FW(9Jszc!aGjx?x~|R?}a&c|fepP%2A1F0IYoAV$9QdM$3`a!3 z)pK=``}9Ptp4tFIW+E~g@vCcX4l``7&c^&?|NDsxYw1YrpJKpwDrSsQ8D{$_P z8F+fk96T>D2mQvGgLu3c6s<7^QMSfl=2lZMX}%Hom$5l`{KXW!C@}$zz#M$7?F;;K zlR&|jLv-|vCNN-WB3LCF4+8Xyz#^p>x?}SMaPY1LVEkoh^P2?7UJs*^Ov7oho(8=; z*_En`vjDYY;sNNBL$7sKrwyF*>6>TbX?ykK^ckmo+V^WNU8NF3DfcU(N6$*37pI?~ zr)y}@kCHFbr)28suHPG*KOCg%?{B4N&glo(yDLCMsth&!{6ktHr5vmb&878sbf-hcC_mAt`Ni z(MVb5V0HpvuCD~itK=CULza>~e;q71-iKP#URSTG)0SZ3C`(XaYY8gXjR)%|S^{;Meqi#p>-4?j{eVx=X^>!T z1=J%(fT7nd!Goy<;JD8*(7tggXq{^TjMiFo_u00@s=J)|mVhBuUqHb&OCT@18|1d zML-AN_Xc6WoGQCG2{gA%01Yx8fKl)OgTg0*qS#$vZ;3m2Kgu2GiN^zx<0uf}G7&^) zj|Ub_V}WMCM3A052E;5V0Y2Mn>1QwHDUXa|5PoVsaCp}O6mMkH4_G|a zI;9=JC3`5*yj}D=6(_pIeFN5SMrb+qprC;F>L1Ks#8mlj^RMwg~trH3uL zLi?BIP}5EB(lP3#^dt9s^gwNAx~5#fm}v=_u3XRAR!eISI7UCd;tfKa(!jSpR@Ca^N^i?f0<-Rv(f&(M({x{1s_sSxz2=rG zV|})XwrYM%OUMOK@}Li65^KS1ol^kb+NXo~{Wi>R!^5b&lL{0{7)*gH6F~fHWhOJ= z8+}Ie60q@I1FnDc29Ae-^7p0VP8(aDK-IFwnybZ1wO4 z4_n2|VRjPOJLeVfeKZL))dhoP>fT_+cXxrO#clA~HWk!}J%O^VXLp}Fy?naMk=pIx zD>S3N9`*vYYL#Gpu^9zEc!TVGZ{Qm-7QF1tiH)ZRXdCPe5>|PFIRkyb{+dAGz<7h? zg`S{3))TZjjsrj5G^1N5M1#+^6Q~Pc<2vVnXkff?Gl-1d41Uww45G?0*f)3sFsxb+ zS|>+=U4zzv+!xW{_W3BVV&x{#S9LQWTQ-71^{1eCfRHIPa-b$MPeIhsL*VmBP3rus zI$H79!yu<32CUVYMC&3mD#_|BmF0Yzjw<$}MeV6nR6q`}B#S>dZ}tAo_r(1*7!RyxRdU0})gD@fNK& zAr4%7Q3Jjn`;My1aAH)~=u@LrWSR6tmTByxLKm3UP`_q1(l)S=ma{ksScOOQ@`Eqv z?WF^#vDVe}t65G==sg9dgVAO_W{Rki?Sq;6n-iFdjSqp&g^xKBzOLva7tPRiUv!Y}Wy76uqf{1#w{Apg0gWbQ9?Odlqx3Y%_4a8Vhcw zbl$_*c;NhbH~9H>EO3mE0m11pKv=N}JkR!^ziQ@#qZ6YjVaEw@W?ms6Sw+Ahw+NgT z6#=g&LWx2GKpz0 z^X_(FrJ^nKX|pQx%abxt)>na=ZI(#sA6UT1t(!=_erm_Cb0V0-ts`hBH8H*X`##L~ z9gk?Ed!2b3pvz!WL+0+P`PA%FA||40J|ppWWX5;6F>BKYQ^w_f%oao?Yj)%IX?h7P5+|IiEu8JqyRq^^6MJQ^#2aBndv zZYu#xpO=F4fu&%{hZ1nZwH2InDFtJvlmR=hnM~aL6Cj`Vp@y5517DNNKu)y`)OjMx z=?hEQPL`w4g<|mfO>uXhGCRt<%Z2oJlupMM>eJp*@cja7YUIT~|(d|W`-k}V%PbdM+A?3h!KrL{FWuOI=fb?l4z7JI|81Rq zj%C-Z+q2f$v-i*E|4$kC_jS#l6*{4?X9aF!Z!)CiAJ@k%I_m-g{TBNM&Hb+_>sl`s zbyMnsfF)sJ^FsW>mxTFBm;JhHdg}SR((1e=v)*d{^-_nP1%H8s#vgn7<9g}-?)6gX z3iclKO2{U|K?TCcO`!RZp0tG>dI-ryv5;R z^SJJKcm8fYePz(x@UGQx={5Ko|6N;tuY7vHQDye0?*8ai_J8o|PmTUL_$LGZYZ(}g zkb{GR%MeGND+&ct!ik3val}v_LWU!T@es07!oh`y5OYNDgZ6KnoRk>i@NacJC*(O| z2oIqkB^>x-QRE1|Sd=(oIL}r3aD+1tq0A9{v8ZqaUo5H|!8f*Q9KkoX>OI5|2VOrk zID)SaO^)EpsTN1@qi7mEo;@cCiN5qx87)=N0@`eDuye0^AO1fL(49Kn~Wu;mE8SnN20&yN8d!RN<7j^Ojdo+J2T8N?BMehlUazOi-aC5H0) zF@z)d`f%h3K0ll|f-k2-r39}Z!#IM^kKs~+*AHio;LE9tl;HJa1V`}20yu)t4~irB z{Gd64&ku$p_+oM82tGeXas=Plx^)vn{=@qR%MpBiK#t(^191djPDe=zUO(J9g3pi9 zQi9iyF&x2{)3H*5*N<@=!57PTj^OiS0!Q%qF_9zq{P5rizF0gtg3k{xj^Gw6g`LRMu@cOZmBlvQ_JVu|1g zK0nrQ1fL&~9Kq+uT8`k0WgSQG`LUiO_{Mfa57GG-5Wf2m2*!>Q2rjq^9**fL>w3$Q Hp7MVHi`i3! diff --git a/old_tests/simulate-chime/data/update_gain.hdf5 b/old_tests/simulate-chime/data/update_gain.hdf5 deleted file mode 100644 index 582bd9c4b84086f553657d1df363100ec4ccb98c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14762 zcmeI22UHYEpTLJ<$QdN(93*3|?t!U7z=VitT_Xwt1`q`m6#)eVK~O=Ah_HgFm{3<; zM3^d2Q8B<8Fe?H^6cABBJ<{}j>$m;wy}Nt&-ra5U-c)zhuWIW5uis2p&m+mp-9tfE zN0!rbNK12g9Od3q(aZP##6)t9233AyNhweIY#|`z5YT#>3+_r z-*3_9ny**UB`JSU^O#ivR&fsu@ zdawVdUc9dl1(!OcK=H#jBX(QZg6KFK(2}hSV4xAW<^5&EgA*2@pxP2le&_%`td;@B z@;1Ov)gD|fk^r#-1_IlGGwC=>T}ox%dC)e-6TmfeRlf<}Orx*QPgoldt!VQNRJfbP7hNmok6(lw8E(pq{Mbj5dH z%6#@VdVSJ7dRwjs9Zl__Or!GX(yK@4p8WVsT~Ft==+g`2B*CVR`QYa0MsVraIa9^BD>6(=TKzYf}w87_jv@o{}%y{NQU+g?eznv39 zpADEp`xNd0j)$}8L$c@SDdWfTp?@&OmXU7dK~u4)6`&g~e{GiH;`;NG!6M%)IL zT0H6zZ~$Hx3cyS=32MkHS8%9vAQ*nTX~cxjk#zhbTflRl0kWd(fzvJ*u>0g_z`N@R zZp2E0&#Bg6%M}X%^0rf5A=^Oy-jiUl%NEdNHU@Z)9|*4aB!C##cn}p80vf(82NO6$ zLGHR(Fx__@7%I6D44O6$Bt3`&v|l1P zq4e0QZNU6O65tJ6LTf1)QcDhgpzd-r>Bhv7^svVX)Vm|;;K=+gpfltay-Z4l&Z&!~ zZ+{v?&%b$!uJo&=+dXUOYSSEQ)xO{9cXzUB{he{NuU#WG?-^_d6h2TSkK)P@4u-*5^ri`qel zFU+Ey{RdETYtm_D*ETxjz^`zQ7bJILA2Kk+QaTKo$)dWoZ6lW%o{g?4gQI}>*QRq38>p|1hv|^z^O6?)E0Sx zu1y<&wRj3xr;rSIM^nI@tStJdT`E{L(SR~I5Dx_LIbd323b>G_O`RMvfO>rA1`sC1 zfd;W*yn$xbbs4Civ3{VgE}xH|24>u$Ohvk)q{649@6yS zN^1W6RaB%&BM2S8jyjipn67Jo56*PH1F+>cdf~kU>RGvf7M^@YyOwOG(^Vc(#+@9h zDt#)oDoT%UY9B;@Z$CvR3sUJ=!sENoFy@bXZ@`}rFF}8cHRKDoyrEZbJwuQFHiQ<_ zvE?f|y6}5G8{uE1`Mwh~Xfs|m2sx}t9c|r8B~GFFLq{&A!r#dA8yE2TZuhn6wJ+Of zg+oUCjr12<>eesd=gp7lJ&&H#7GGCVP4Dj0^UMbEBR)y<4_4fyS8o_YO&qSvcaa*# zS9>y&a=&UxeK|6UUy@ZpDFww+ztp5saUWlSHLJ||r)Qb)Wh&o+s~_9InM?0M?!i~R z>m*&=28N|KfxMM+l*hGJP$Sp}rX7C?O0`>nf9wY^?@cREsBWe+)Z4-FXK|F$iNAg4VU!yO67UzeN8+4PeH{UvL zEk7$hnlB!-l8)FO%`X)n#s_zKeCKU@>F3-H{Ata}y$;AMo63J;UQ6c`$x(i0Td7@? z3_Tlc<$-#m+{_0D!lve^Ik|*Slyu{ef8)X=@@G6FLS8zPwIeam^I zj-a*{Z=mujL#V4GR(?x)A=mr_kbo>Kqy{ZUNsUPhz)d9G(?v$Q7( z|AFJr_EW+A{X}~(wqgFPFZ({stAERD^oJevKkmbVd-q{|@Al96=<6;f*0W>d^zQ-x zy%gvCG5&c0A>m5{`o{WKuYVT9ieUfnp55SoZ~Q@4ee*E-!*u>J{zbj<{~8L4F~#gZ=+9e?9r{`)Ng) zGcR=M(tt&N;h{@siyljR`~UFM%Rh~&)~&ixe=YmxDzV=ylRp^ok6tbL7he6-oBmz# zcLx5>!2fs#G!2bNz%fOk6AdJZ$w5`}@~jV50bhi=ellocKr)^{$s?}3 z8tS@GB((M6A*0BXaQ=21G{++vFEC4lU6RshO{F$`Rf0~jvkqBp)9V2+kDD(}=Llwg#baYQ(7D*g_O5P>GeW$=6aCpfTmDLx#< zLldq;NT!{J{&HO6w73a6<~+tz%^ch^FIysCK^NQ=yB*r&`!JZgg+&XjxUUnxC@98D zhn{dV{BA{-t~v_Oh+ClgFTU{bdUaHcvC#a96*_F+0rOv{!-UWdIMt>Sj*)G^x6WS0 zV4?)U`_pkrgA!SinT;Q|4o3TzRtsgf)WJcgE8&B9XaF>Z1+M{g)&)I0BqF-guWtE9#^~;P& zV$N8sP$Nrn4jy3gr7c}mHsGxlx^ztz<$c=)&pkK~+m1HiGHnKho^S@R?Q*nY9TF4P)>8-fx zkvWZ72yt#Dq26d5vE z4w+3?#P_AukVV@8SZe8j2fiIZ#y9JctjP0a@^=TK5A#;xEha_;ZfJ)2x%O^n?d-|x zn*_VA=!7$X9a;4534TysB4}OfMIJ54!!;lIgr39^%+0q#qlTs9fjg{_-072$Z@?uV zqD$e9F^mse$XsQcW)(t>s);Gc^FM@A;GeU8%6NJU(InZ#DBpLTinM6(;i`TDsD*Re0 zgO+H=2LO9NDXPvAoR<#=9Bo?Sk()AIXov%eC{nCXB+gIWjlf_A9M+FWU+zijwh?9^viLiEx z5tN!~f?miOkj543V7`hBa$b20*Q|)=&&jkPp(?E~W&I`is?t%I;jM)9%(;j=p74*m(Yv`L~p4*X80>;KSdm6cx=F{U&Z28Tro0VWdH(Ip9GI`CD8aWJ7NA(LsSS6&Of~l?o!r6 z$~8?ecDNL|-hH3#^wq*%r&Q2`%6+gsWdt6stVZT;)h6rP?8}NGERoCIXdE7?OC}^W z!@Z@ZC3Z=+WZEW-<==mSw>b7B;MOB7UUotd)$TzwGV}1zFpAi(A1kPkw?->}*^G1E zS)flN4#9i1T(Y}P2!CGQ4!;gBfODlk;>jO1$XnOt!p|A)uH=IgT7T*T45>PX3zYPT zj5b%WvbX@>T>cx5Ixm5&-i*LAf=!^T?Epk^>fzJ6axDC4h*Xd46vl7g1wXEkBnv(( zlCLG6xEWp)7MaN+n;UCzfV3Q{%vC~b0^SKT;v|r`ZXs;?YKh#$6L3PwHn`A(hdvid zz@lm{IVN@+=kGcW6O0~Vr?OMHVB!~e!9E+}hHQNHI2TplPlgpE6cOjF7V$4T=H zCp_1Y2Ir{Xg>x7F&iJwRco+V<{3O0MCc!Q4r6t*xbrRmbXo!mMPlG92HPDB9$Am+g z%urcd2MpMk3O!oBz*6;MxE8;}xmg!+va+*!bhadNuISK%o=P#k~LG|w6})Dd-zoBuxJ2DHq63v8kA6wnIc;8 zCQVo$CPftXEXR#a23T#360$t>yFez#7I79Wj!olmZ`fY~g7=V`P0nha9;T4?k~_Kxw0H<5H`IWl=t6L|>s5*6$$j`XK|M zUyvHQre+V-iqz2ze>pVwx7sqOd7P> zaSm=3Yrt_+n_$%SOE~NVmsHE#!cTrRBlY?_@t4GA>~X#r*FG@B)Hq49BdG&h)lY!q z@-E_f-%=Pq?E;L-yo`-x707=^3@5to zse(nbtx@LVO?bJvIa+AH57HUpByDmzG<9u(ZKDeyUEPMwnl;FkPrAa*8{*xT?{z?t zd)~uT*gJB`L7`6)u>ehb+d= zgr6id5l1{ZuNWqRq*EpN-Hy*^o?An~k47RV7wt zJMpv*MO2uofIKwPg)0oD$Zdl~c-}rOEID5hEy=hqh}XA9wu(B$Y~e^aCcgVwqj{~c zI4BM{D5)7;nLsg&_A6@jNZ1p{3 z;?vv$i>8-Cr;&0(^H4QZ)#w1d%GA&mS9!Gef~@cfPlvp%vch1a5~-V;#^meBCU>{H z93!-HvlQygPl8h}oPnI+dhGF{4yw$*g8iq6lkpjqm|tQ-H0x7wl5ZOhbUBVK9+>07 zPbJ8M=j|9u`N8dJ7@J8DsPOR&T)yBce)>g$gtdp_DVn;(RD3tQyt+XUa#DpT9QGGp z9?3<#xgCNR-FupVdM*j8tr6T>y#ViQ;~^D~aO^%$2epm1K-Z#c zs-tLAT@Rdl`1^bVvPJKD*#M~;yn0y^)Eme{8UgXR++e@W28^)F8P9y6I2E zWseoR=kMjt;aui(8a?~U>;2{F{*vQ?kfWpHFehhGs6Uiq4B_%ai|CTe5Q9WjKa}DO zG5CiT(IrnrI1Uq4{ZL9UME}1;{?N|p{!bB6ERO%w)>k3L5Kb(Fw1{wIi$#VZ*kX}o zh#@TRBF7LzSqOQCV2ee8A=qM3WC*skRbmLXwN>sToE%yGP+!Iskj zB7)Tqb%tQ`LqkM3vE@{gA=q-NB_de;&}Im>SacYI%@19MVDm$dA=qNkX9%`f3>bpV z4?~7vTU(=k!kN_%V}@WG!-OH&{4iw*ww#)Y2v$GL8G_9Z3lYKUhb2R><EeO{~=%qwlN?>u=#-)f-R@+B7)VAkqp7+hlhw@^}~}P*mCM6B3S(x z#Sm<F_$6O{FuiOY&rE85v+a$Fa(<)^F;)!AAtPI9)u*I^9A=vy_%@AyUtYHW?KcX0d ZEta(m!RALaL$IxFOdrwx3kcgh{ta_ukwyRj diff --git a/old_tests/simulate-chime/docker-compose.yaml b/old_tests/simulate-chime/docker-compose.yaml deleted file mode 100644 index 271a4067..00000000 --- a/old_tests/simulate-chime/docker-compose.yaml +++ /dev/null @@ -1,60 +0,0 @@ -# Docker compose configuration defining gpu/receiver node containers -version: '2.3' -# Need to define networks again because Docker ignores them in base.yaml -networks: - receiver_net: - driver: bridge - ipam: - driver: default - config: - - subnet: 10.0.0.0/16 -services: - comet: - build: comet - networks: - - receiver_net - - default - #- comet_net - # should overwrite these in compose file - hostname: comet - image: comet - ports: - - "12050:12050" - networks: - receiver_net: - # address needs to match that in kotekan config yaml's - ipv4_address: 10.0.1.3 - gpu-cn01: - image: gpu-cn01 # this allows image to be reused without rebuilding - volumes: - - ./:/test - extends: - file: base.yaml - service: gpu - environment: - KOTEKAN_CONFIG: /test/gpu.yaml - hostname: cn01 - ports: - - "12100-12109:12048" - restart: on-failure - recv-1: - image: recv-1 - volumes: - - ./:/test - extends: - file: base.yaml - service: receiver - environment: - KOTEKAN_CONFIG: /test/recv.yaml - RUN_DATASET_BROKER: 1 - hostname: recv-1 - ports: - - "12049:12048" - networks: - receiver_net: - # address needs to match that in receiver config yaml - ipv4_address: 10.0.1.2 - restart: on-failure - # for running interactive container - #stdin_open: true - #tty: true diff --git a/old_tests/simulate-chime/gps_server.json b/old_tests/simulate-chime/gps_server.json deleted file mode 100644 index 6ee83d10..00000000 --- a/old_tests/simulate-chime/gps_server.json +++ /dev/null @@ -1 +0,0 @@ -{"frame0_time": [2019, 6, 24, 15, 35, 40, 999999.96], "frame0_nano": 1561390540999999960, "frame0_ctime": 1561390541.0, "start_ctime": 1561389923.256754} diff --git a/old_tests/simulate-chime/gpu.yaml b/old_tests/simulate-chime/gpu.yaml deleted file mode 100644 index a8ed6f38..00000000 --- a/old_tests/simulate-chime/gpu.yaml +++ /dev/null @@ -1,790 +0,0 @@ -########################################## -# -# gpu.yaml -# -# A copy of the CHIME project GPU config, altered to be used with docker: -# 128 elements, and 4 frequencies. -# -# The actual GPU part, the FRB and tracking part is excluded, since there is no -# way to simulate it on CPU. -# -# Author: Andre Renard, Rick Nitsche -# -########################################## -type: config -# Logging level can be one of: -# OFF, ERROR, WARN, INFO, DEBUG, DEBUG2 (case insensitive) -# Note DEBUG and DEBUG2 require a build with (-DCMAKE_BUILD_TYPE=Debug) -log_level: info -num_elements: 128 -num_local_freq: 1 -num_data_sets: 1 -samples_per_data_set: 49152 -buffer_depth: 2 -baseband_buffer_depth: 282 # 282 = ~34 seconds after accounting for active frames -vbuffer_depth: 32 -num_links: 4 -timesamples_per_packet: 2 -# cpu_affinity: [4,5,6,7,8,9,10,11] -cpu_affinity: [2,3,8,9] -block_size: 32 -num_gpus: 4 -link_map: [0,1,2,3] - -dataset_manager: - use_dataset_broker: True - ds_broker_host: "10.0.1.3" - ds_broker_port: 12050 - -# Constants -sizeof_float: 4 -sizeof_short: 2 - -# N2 global options -num_ev: 4 -# This option now does very little. You probably want to look at -# visAccumulate:integration_time -num_gpu_frames: 128 -# Sets the number of sub frames for shorter than ~120ms N2 output -# Please note this requires changing the number of commands in the -# GPU section, and the accumulate value for `samples_per_data_set` -num_sub_frames: 4 - -# N2 global options -num_ev: 4 -# This option now does very little. You probably want to look at -# visAccumulate:integration_time -num_gpu_frames: 128 -# Sets the number of sub frames for shorter than ~120ms N2 output -# Please note this requires changing the number of commands in the -# GPU section, and the accumulate value for `samples_per_data_set` -num_sub_frames: 4 - -#FRB global options -downsample_time: 3 -downsample_freq: 8 -factor_upchan: 128 -factor_upchan_out: 16 -num_frb_total_beams: 1024 -frb_missing_gains: [1.0,1.0] -frb_scaling: 0.05 #1.0 -reorder_map: [32,33,34,35,40,41,42,43,48,49,50,51,56,57,58,59,96,97,98,99,104,105,106,107,112,113,114,115,120,121,122,123,67,66,65,64,75,74,73,72,83,82,81,80,91,90,89,88,3,2,1,0,11,10,9,8,19,18,17,16,27,26,25,24,152,153,154,155,144,145,146,147,136,137,138,139,128,129,130,131,216,217,218,219,208,209,210,211,200,201,202,203,192,193,194,195,251,250,249,248,243,242,241,240,235,234,233,232,227,226,225,224,187,186,185,184,179,178,177,176,171,170,169,168,163,162,161,160,355,354,353,352,363,362,361,360,371,370,369,368,379,378,377,376,291,290,289,288,299,298,297,296,307,306,305,304,315,314,313,312,259,258,257,256,264,265,266,267,272,273,274,275,280,281,282,283,323,322,321,320,331,330,329,328,339,338,337,336,347,346,345,344,408,409,410,411,400,401,402,403,392,393,394,395,384,385,386,387,472,473,474,475,464,465,466,467,456,457,458,459,448,449,450,451,440,441,442,443,432,433,434,435,424,425,426,427,416,417,418,419,504,505,506,507,496,497,498,499,488,489,490,491,480,481,482,483,36,37,38,39,44,45,46,47,52,53,54,55,60,61,62,63,100,101,102,103,108,109,110,111,116,117,118,119,124,125,126,127,71,70,69,68,79,78,77,76,87,86,85,84,95,94,93,92,7,6,5,4,15,14,13,12,23,22,21,20,31,30,29,28,156,157,158,159,148,149,150,151,140,141,142,143,132,133,134,135,220,221,222,223,212,213,214,215,204,205,206,207,196,197,198,199,255,254,253,252,247,246,245,244,239,238,237,236,231,230,229,228,191,190,189,188,183,182,181,180,175,174,173,172,167,166,165,164,359,358,357,356,367,366,365,364,375,374,373,372,383,382,381,380,295,294,293,292,303,302,301,300,311,310,309,308,319,318,317,316,263,262,261,260,268,269,270,271,276,277,278,279,284,285,286,287,327,326,325,324,335,334,333,332,343,342,341,340,351,350,349,348,412,413,414,415,404,405,406,407,396,397,398,399,388,389,390,391,476,477,478,479,468,469,470,471,460,461,462,463,452,453,454,455,444,445,446,447,436,437,438,439,428,429,430,431,420,421,422,423,508,509,510,511,500,501,502,503,492,493,494,495,484,485,486,487] -frb_gain: - kotekan_update_endpoint: json - frb_gain_dir: /mnt/gong/cherry/FRB-GainFiles/broker_CygA_10_26_scl/ -frb_gai: - kotekan_update_endpoint: json - frb_gain_dir: /mnt/gong/cherry/FRB-GainFiles/broker_CygA_10_26_scl/ - -# Fake the endpoints from all stages we can't run in docker using the updatable config functionality -frb: - update_beam_offset: - kotekan_update_endpoint: json - beam_offset: 108 -gpu: - gpu_0: - frb: - update_EW_beam: - 0: - kotekan_update_endpoint: json - ew_id: 1 - ew_beam: 0.1 - update_NS_beam: - 0: - kotekan_update_endpoint: json - northmost_beam: 0.5 - update_bad_inputs: - kotekan_update_endpoint: json - bad_inputs: [1, 2, 3] - update_tracking: - 0: - kotekan_update_endpoint: json - beam: 1 - ra: [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0] - dec: [1.0, 0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1] - scaling: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - gpu_1: - frb: - update_EW_beam: - 1: - kotekan_update_endpoint: json - ew_id: 1 - ew_beam: 0.1 - update_NS_beam: - 1: - kotekan_update_endpoint: json - northmost_beam: 0.5 - update_bad_inputs: - kotekan_update_endpoint: json - bad_inputs: [1, 2, 3] - update_tracking: - 0: - kotekan_update_endpoint: json - beam: 1 - ra: [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0] - dec: [1.0, 0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1] - scaling: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - gpu_2: - frb: - update_EW_beam: - 2: - kotekan_update_endpoint: json - ew_id: 1 - ew_beam: 0.1 - update_NS_beam: - 2: - kotekan_update_endpoint: json - northmost_beam: 0.5 - update_bad_inputs: - kotekan_update_endpoint: json - bad_inputs: [1, 2, 3] - update_tracking: - 0: - kotekan_update_endpoint: json - beam: 1 - ra: [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0] - dec: [1.0, 0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1] - scaling: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - gpu_3: - frb: - update_EW_beam: - 3: - kotekan_update_endpoint: json - ew_id: 1 - ew_beam: 0.1 - update_NS_beam: - 3: - kotekan_update_endpoint: json - northmost_beam: 0.5 - update_bad_inputs: - kotekan_update_endpoint: json - bad_inputs: [1, 2, 3] - update_tracking: - 0: - kotekan_update_endpoint: json - beam: 1 - ra: [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0] - dec: [1.0, 0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1] - scaling: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - -#Pulsar stuff -feed_sep_NS : 0.3048 -feed_sep_EW : 22.0 -num_beams: 10 -num_pol: 2 -tracking_gain: - 0: - kotekan_update_endpoint: json - gain_dir: /mnt/gong/cherry/FRB-GainFiles/broker_CygA_10_26_scl/ - 1: - kotekan_update_endpoint: json - gain_dir: /mnt/gong/cherry/FRB-GainFiles/broker_CygA_10_26_scl/ - 2: - kotekan_update_endpoint: json - gain_dir: /mnt/gong/cherry/FRB-GainFiles/broker_CygA_10_26_scl/ - 3: - kotekan_update_endpoint: json - gain_dir: /mnt/gong/cherry/FRB-GainFiles/broker_CygA_10_26_scl/ - 4: - kotekan_update_endpoint: json - gain_dir: /mnt/gong/cherry/FRB-GainFiles/broker_CygA_10_26_scl/ - 5: - kotekan_update_endpoint: json - gain_dir: /mnt/gong/cherry/FRB-GainFiles/broker_CygA_10_26_scl/ - 6: - kotekan_update_endpoint: json - gain_dir: /mnt/gong/cherry/FRB-GainFiles/broker_CygA_10_26_scl/ - 7: - kotekan_update_endpoint: json - gain_dir: /mnt/gong/cherry/FRB-GainFiles/broker_CygA_10_26_scl/ - 8: - kotekan_update_endpoint: json - gain_dir: /mnt/gong/cherry/FRB-GainFiles/broker_CygA_10_26_scl/ - 9: - kotekan_update_endpoint: json - gain_dir: /mnt/gong/cherry/FRB-GainFiles/broker_CygA_10_26_scl/ - -#RFI stuff -sk_step: 256 -rfi_combined: True -rfi_sigma_cut: 5 - -#RFI Live-view Paramters -waterfallX: 1024 -num_receive_threads: 4 -colorscale: 0.028 -waterfall_request_delay: 60 - -rfi_masking: - toggle: - kotekan_update_endpoint: "json" - rfi_zeroing: False - - -# Pool -main_pool: - kotekan_metadata_pool: chimeMetadata - num_metadata_objects: 30 * buffer_depth + 5 * baseband_buffer_depth - -gpu_n2_buffers: - # We need a longer output buffer depth because the - # output now comes in chunks of 4 at once. - num_frames: buffer_depth * 2 - frame_size: 4 * num_data_sets * num_local_freq * ((num_elements * num_elements) + (num_elements * block_size)) - metadata_pool: main_pool - gpu_n2_output_buffer_0: - kotekan_buffer: standard - gpu_n2_output_buffer_1: - kotekan_buffer: standard - gpu_n2_output_buffer_2: - kotekan_buffer: standard - gpu_n2_output_buffer_3: - kotekan_buffer: standard - valve_buffer_0: - kotekan_buffer: standard - valve_buffer_1: - kotekan_buffer: standard - valve_buffer_2: - kotekan_buffer: standard - valve_buffer_3: - kotekan_buffer: standard - -gpu_beamform_output_buffers: - num_frames: buffer_depth - frame_size: num_data_sets * (samples_per_data_set/downsample_time/downsample_freq) * num_frb_total_beams * sizeof_float - metadata_pool: main_pool - gpu_beamform_output_buffer_0: - kotekan_buffer: standard - gpu_beamform_output_buffer_1: - kotekan_buffer: standard - gpu_beamform_output_buffer_2: - kotekan_buffer: standard - gpu_beamform_output_buffer_3: - kotekan_buffer: standard - -tracking_output_buffers: - num_frames: buffer_depth - frame_size: _samples_per_data_set * _num_beams * _num_pol * sizeof_float *2 - metadata_pool: main_pool - beamform_tracking_output_buffer_0: - kotekan_buffer: standard - beamform_tracking_output_buffer_1: - kotekan_buffer: standard - beamform_tracking_output_buffer_2: - kotekan_buffer: standard - beamform_tracking_output_buffer_3: - kotekan_buffer: standard - -# Metadata pool -vis_pool: - kotekan_metadata_pool: visMetadata - num_metadata_objects: 200 * buffer_depth - -# Buffers -vis_buffers: - metadata_pool: vis_pool - num_frames: buffer_depth - visbuf_5s_0: - kotekan_buffer: vis - visbuf_5s_1: - kotekan_buffer: vis - visbuf_5s_2: - kotekan_buffer: vis - visbuf_5s_3: - kotekan_buffer: vis - visbuf_5s_merge: - kotekan_buffer: vis - visbuf_10s_0: - kotekan_buffer: vis - visbuf_10s_1: - kotekan_buffer: vis - visbuf_10s_2: - kotekan_buffer: vis - visbuf_10s_3: - kotekan_buffer: vis - visbuf_10s_merge: - num_frames: 8 - kotekan_buffer: vis - # Buffers for gated - visbuf_psr0_5s_0: - kotekan_buffer: vis - visbuf_psr0_5s_1: - kotekan_buffer: vis - visbuf_psr0_5s_2: - kotekan_buffer: vis - visbuf_psr0_5s_3: - kotekan_buffer: vis - visbuf_psr0_5s_merge: - kotekan_buffer: vis - # Increase the buffer depth for the pre-send buffers - visbuf_5s_26m: - kotekan_buffer: vis - num_prod: 4096 - num_frames: 10 * buffer_depth - visbuf_psr0_5s_26m: - kotekan_buffer: vis - num_prod: 4096 - num_frames: 10 * buffer_depth - -gpu_rfi_output_buffers: - num_frames: buffer_depth - frame_size: sizeof_float * num_local_freq * samples_per_data_set / sk_step - metadata_pool: main_pool - gpu_rfi_output_buffer_0: - kotekan_buffer: standard - gpu_rfi_output_buffer_1: - kotekan_buffer: standard - gpu_rfi_output_buffer_2: - kotekan_buffer: standard - gpu_rfi_output_buffer_3: - kotekan_buffer: standard - -gpu_rfi_mask_output_buffers: - num_frames: buffer_depth - frame_size: num_local_freq * samples_per_data_set / sk_step - metadata_pool: main_pool - gpu_rfi_mask_output_buffer_0: - kotekan_buffer: standard - gpu_rfi_mask_output_buffer_1: - kotekan_buffer: standard - gpu_rfi_mask_output_buffer_2: - kotekan_buffer: standard - gpu_rfi_mask_output_buffer_3: - kotekan_buffer: standard - -gpu_rfi_bad_input_buffers: - num_frames: buffer_depth - frame_size: sizeof_float * num_elements * num_local_freq - metadata_pool: main_pool - gpu_rfi_bad_input_buffer_0: - kotekan_buffer: standard - gpu_rfi_bad_input_buffer_1: - kotekan_buffer: standard - gpu_rfi_bad_input_buffer_2: - kotekan_buffer: standard - gpu_rfi_bad_input_buffer_3: - kotekan_buffer: standard - -cpu_affinity: [2,3,8,9] -fake_dpdk_0: - kotekan_stage: fakeGpuBuffer - out_buf: gpu_n2_output_buffer_0 - wait: True - mode: "gaussian" - freq: 0 - -fake_dpdk_1: - kotekan_stage: fakeGpuBuffer - out_buf: gpu_n2_output_buffer_1 - wait: True - mode: "gaussian" - freq: 1 - -fake_dpdk_2: - kotekan_stage: fakeGpuBuffer - out_buf: gpu_n2_output_buffer_2 - wait: True - mode: "gaussian" - freq: 2 - -fake_dpdk_3: - kotekan_stage: fakeGpuBuffer - out_buf: gpu_n2_output_buffer_3 - wait: True - mode: "gaussian" - freq: 3 - -#### N2 GPU Post processing and Tx #### - -# Updatable config for gating -updatable_config: - gating: - psr0_config: - kotekan_update_endpoint: "json" - enabled: false - # B0329 (2018/11/14) - pulsar_name: "B0329" - pulse_width: 0.0314 - dm: 26.7641 - segment: 18000. - rot_freq: 1.39954153872 - t_ref: [58437.4583333332,] - phase_ref: [1446745468.8439252,] - coeff: [[ - 3.428779943504219e-10, - 0.0011253963075329334, - -1.1565642124857199e-07, - 1.1347089844281842e-10, - 1.1882399863116445e-13, - -1.0867680647236115e-16, - -7.594309559679319e-20, - 5.042548967792808e-23, - 2.745973613190195e-26, - -2.279050323988331e-29, - -1.1771713330451292e-32, - 3.948826407949732e-35, - ],] - -valve: - valve0: - kotekan_stage: Valve - in_buf: gpu_n2_output_buffer_0 - out_buf: valve_buffer_0 - valve1: - kotekan_stage: Valve - in_buf: gpu_n2_output_buffer_1 - out_buf: valve_buffer_1 - valve2: - kotekan_stage: Valve - in_buf: gpu_n2_output_buffer_2 - out_buf: valve_buffer_2 - valve3: - kotekan_stage: Valve - in_buf: gpu_n2_output_buffer_3 - out_buf: valve_buffer_3 - -vis_accumulate: - integration_time: 5.0 # Integrate to roughly 5s cadence - # This (12288) is for num_sub_frames = 4, there is currently a config bug that - # prevents referencing the higher level samples_per_data_set and dividing it - samples_per_data_set: 12288 - acc0: - kotekan_stage: visAccumulate - in_buf: valve_buffer_0 - out_buf: visbuf_5s_0 - gating: - psr0: - mode: pulsar - buf: visbuf_psr0_5s_0 - updatable_config: - psr0: "/updatable_config/gating/psr0_config" - acc1: - kotekan_stage: visAccumulate - in_buf: valve_buffer_1 - out_buf: visbuf_5s_1 - gating: - psr0: - mode: pulsar - buf: visbuf_psr0_5s_1 - updatable_config: - psr0: "/updatable_config/gating/psr0_config" - acc2: - kotekan_stage: visAccumulate - in_buf: valve_buffer_2 - out_buf: visbuf_5s_2 - gating: - psr0: - mode: pulsar - buf: visbuf_psr0_5s_2 - updatable_config: - psr0: "/updatable_config/gating/psr0_config" - acc3: - kotekan_stage: visAccumulate - in_buf: valve_buffer_3 - out_buf: visbuf_5s_3 - gating: - psr0: - mode: pulsar - buf: visbuf_psr0_5s_3 - updatable_config: - psr0: "/updatable_config/gating/psr0_config" - -## Perform all extra time integration (skip calculate eigenvalues) - -vis_merge_5s: - kotekan_stage: bufferMerge - timeout: 0.1 - in_bufs: - - visbuf_5s_0 - - visbuf_5s_1 - - visbuf_5s_2 - - visbuf_5s_3 - out_buf: visbuf_5s_merge - -vis_merge_gated_psr0: - kotekan_stage: bufferMerge - timeout: 0.1 - in_bufs: - - visbuf_psr0_5s_0 - - visbuf_psr0_5s_1 - - visbuf_psr0_5s_2 - - visbuf_psr0_5s_3 - out_buf: visbuf_psr0_5s_merge - -vis_int_10s: - num_samples: 2 - int0: - kotekan_stage: timeDownsample - in_buf: visbuf_5s_0 - out_buf: visbuf_10s_0 - int1: - kotekan_stage: timeDownsample - in_buf: visbuf_5s_1 - out_buf: visbuf_10s_1 - int2: - kotekan_stage: timeDownsample - in_buf: visbuf_5s_2 - out_buf: visbuf_10s_2 - int3: - kotekan_stage: timeDownsample - in_buf: visbuf_5s_3 - out_buf: visbuf_10s_3 - -vis_merge_10s: - kotekan_stage: bufferMerge - timeout: 0.1 - in_bufs: - - visbuf_10s_0 - - visbuf_10s_1 - - visbuf_10s_2 - - visbuf_10s_3 - out_buf: visbuf_10s_merge - -vis_debug: - kotekan_stage: visDebug - in_buf: visbuf_10s_merge - -## Start frequency and baseline downselect - -# Generate the 26m stream -26m_subset: - kotekan_stage: prodSubset - in_buf: visbuf_5s_merge - out_buf: visbuf_5s_26m - prod_subset_type: have_inputs - input_list: [1225, 1521] # 26m channels - -# Generate the 26m gated stream -26m_psr0_subset: - kotekan_stage: prodSubset - in_buf: visbuf_psr0_5s_merge - out_buf: visbuf_psr0_5s_26m - prod_subset_type: have_inputs - input_list: [1225, 1521] # 26m channels - -## End subsetting - -# Transmit all the data to the receiver node -buffer_send: - server_ip: 10.0.1.2 - reconnect_time: 20 - log_level: warn - n2: - kotekan_stage: bufferSend - buf: visbuf_10s_merge - server_port: 11024 - 26m: - kotekan_stage: bufferSend - buf: visbuf_5s_26m - server_port: 11025 - 26m_psr0: - kotekan_stage: bufferSend - buf: visbuf_psr0_5s_26m - server_port: 11026 - -buffer_status: - kotekan_stage: bufferStatus - time_delay: 30000000 - - -rfi_broadcast: - total_links: 1 - destination_protocol: UDP - destination_ip: 10.1.13.1 - gpu_0: - kotekan_stage: rfiBroadcast - rfi_in: gpu_rfi_output_buffer_0 - rfi_mask: gpu_rfi_mask_output_buffer_0 - destination_port: 41215 - frames_per_packet: 1 - gpu_1: - kotekan_stage: rfiBroadcast - rfi_in: gpu_rfi_output_buffer_1 - rfi_mask: gpu_rfi_mask_output_buffer_1 - destination_port: 41216 - frames_per_packet: 1 - gpu_2: - kotekan_stage: rfiBroadcast - rfi_in: gpu_rfi_output_buffer_2 - rfi_mask: gpu_rfi_mask_output_buffer_2 - destination_port: 41217 - frames_per_packet: 1 - gpu_3: - kotekan_stage: rfiBroadcast - rfi_in: gpu_rfi_output_buffer_3 - rfi_mask: gpu_rfi_mask_output_buffer_3 - destination_port: 41218 - frames_per_packet: 1 - -rfi_bad_input_finder: - destination_ip: 10.1.13.1 - destination_port: 41219 - gpu_0: - bi_frames_per_packet: 10 - kotekan_stage: rfiBadInputFinder - rfi_in: gpu_rfi_bad_input_buffer_0 - gpu_1: - bi_frames_per_packet: 10 - kotekan_stage: rfiBadInputFinder - rfi_in: gpu_rfi_bad_input_buffer_1 - gpu_2: - bi_frames_per_packet: 10 - kotekan_stage: rfiBadInputFinder - rfi_in: gpu_rfi_bad_input_buffer_2 - gpu_3: - bi_frames_per_packet: 10 - kotekan_stage: rfiBadInputFinder - rfi_in: gpu_rfi_bad_input_buffer_3 - -rfi_record: - total_links: 1 - gpu_0: - kotekan_stage: rfiRecord - write_to: /mnt/gong/RFI - write_to_disk: false - rfi_in: gpu_rfi_output_buffer_0 - gpu_1: - kotekan_stage: rfiRecord - write_to: /mnt/gong/RFI - write_to_disk: false - rfi_in: gpu_rfi_output_buffer_1 - gpu_2: - kotekan_stage: rfiRecord - write_to: /mnt/gong/RFI - write_to_disk: false - rfi_in: gpu_rfi_output_buffer_2 - gpu_3: - kotekan_stage: rfiRecord - write_to: /mnt/gong/RFI - write_to_disk: false - rfi_in: gpu_rfi_output_buffer_3 - -input_reorder: - - [ 0 , 0 , FCC000000 ] - - [ 1 , 1 , FCC000001 ] - - [ 2 , 2 , FCC000002 ] - - [ 3 , 3 , FCC000003 ] - - [ 4 , 4 , FCC000004 ] - - [ 5 , 5 , FCC000005 ] - - [ 6 , 6 , FCC000006 ] - - [ 7 , 7 , FCC000007 ] - - [ 8 , 8 , FCC000008 ] - - [ 9 , 9 , FCC000009 ] - - [ 10 , 10 , FCC0000010 ] - - [ 11 , 11 , FCC0000011 ] - - [ 12 , 12 , FCC0000012 ] - - [ 13 , 13 , FCC0000013 ] - - [ 14 , 14 , FCC0000014 ] - - [ 15 , 15 , FCC0000015 ] - - [ 16 , 16 , FCC0000016 ] - - [ 17 , 17 , FCC0000017 ] - - [ 18 , 18 , FCC0000018 ] - - [ 19 , 19 , FCC0000019 ] - - [ 20 , 20 , FCC0000020 ] - - [ 21 , 21 , FCC0000021 ] - - [ 22 , 22 , FCC0000022 ] - - [ 23 , 23 , FCC0000023 ] - - [ 24 , 24 , FCC0000024 ] - - [ 25 , 25 , FCC0000025 ] - - [ 26 , 26 , FCC0000026 ] - - [ 27 , 27 , FCC0000027 ] - - [ 28 , 28 , FCC0000028 ] - - [ 29 , 29 , FCC0000029 ] - - [ 30 , 30 , FCC0000030 ] - - [ 31 , 31 , FCC0000031 ] - - [ 32 , 32 , FCC0000032 ] - - [ 33 , 33 , FCC0000033 ] - - [ 34 , 34 , FCC0000034 ] - - [ 35 , 35 , FCC0000035 ] - - [ 36 , 36 , FCC0000036 ] - - [ 37 , 37 , FCC0000037 ] - - [ 38 , 38 , FCC0000038 ] - - [ 39 , 39 , FCC0000039 ] - - [ 40 , 40 , FCC0000040 ] - - [ 41 , 41 , FCC0000041 ] - - [ 42 , 42 , FCC0000042 ] - - [ 43 , 43 , FCC0000043 ] - - [ 44 , 44 , FCC0000044 ] - - [ 45 , 45 , FCC0000045 ] - - [ 46 , 46 , FCC0000046 ] - - [ 47 , 47 , FCC0000047 ] - - [ 48 , 48 , FCC0000048 ] - - [ 49 , 49 , FCC0000049 ] - - [ 50 , 50 , FCC0000050 ] - - [ 51 , 51 , FCC0000051 ] - - [ 52 , 52 , FCC0000052 ] - - [ 53 , 53 , FCC0000053 ] - - [ 54 , 54 , FCC0000054 ] - - [ 55 , 55 , FCC0000055 ] - - [ 56 , 56 , FCC0000056 ] - - [ 57 , 57 , FCC0000057 ] - - [ 58 , 58 , FCC0000058 ] - - [ 59 , 59 , FCC0000059 ] - - [ 60 , 60 , FCC0000060 ] - - [ 61 , 61 , FCC0000061 ] - - [ 62 , 62 , FCC0000062 ] - - [ 63 , 63 , FCC0000063 ] - - [ 64 , 64 , FCC0000064 ] - - [ 65 , 65 , FCC0000065 ] - - [ 66 , 66 , FCC0000066 ] - - [ 67 , 67 , FCC0000067 ] - - [ 68 , 68 , FCC0000068 ] - - [ 69 , 69 , FCC0000069 ] - - [ 70 , 70 , FCC0000070 ] - - [ 71 , 71 , FCC0000071 ] - - [ 72 , 72 , FCC0000072 ] - - [ 73 , 73 , FCC0000073 ] - - [ 74 , 74 , FCC0000074 ] - - [ 75 , 75 , FCC0000075 ] - - [ 76 , 76 , FCC0000076 ] - - [ 77 , 77 , FCC0000077 ] - - [ 78 , 78 , FCC0000078 ] - - [ 79 , 79 , FCC0000079 ] - - [ 80 , 80 , FCC0000080 ] - - [ 81 , 81 , FCC0000081 ] - - [ 82 , 82 , FCC0000082 ] - - [ 83 , 83 , FCC0000083 ] - - [ 84 , 84 , FCC0000084 ] - - [ 85 , 85 , FCC0000085 ] - - [ 86 , 86 , FCC0000086 ] - - [ 87 , 87 , FCC0000087 ] - - [ 88 , 88 , FCC0000088 ] - - [ 89 , 89 , FCC0000089 ] - - [ 90 , 90 , FCC0000090 ] - - [ 91 , 91 , FCC0000091 ] - - [ 92 , 92 , FCC0000092 ] - - [ 93 , 93 , FCC0000093 ] - - [ 94 , 94 , FCC0000094 ] - - [ 95 , 95 , FCC0000095 ] - - [ 96 , 96 , FCC0000096 ] - - [ 97 , 97 , FCC0000097 ] - - [ 98 , 98 , FCC0000098 ] - - [ 99 , 99 , FCC0000099 ] - - [ 100, 100, FCC00000100] - - [ 101, 101, FCC00000101] - - [ 102, 102, FCC00000102] - - [ 103, 103, FCC00000103] - - [ 104, 104, FCC00000104] - - [ 105, 105, FCC00000105] - - [ 106, 106, FCC00000106] - - [ 107, 107, FCC00000107] - - [ 108, 108, FCC00000108] - - [ 109, 109, FCC00000109] - - [ 110, 110, FCC00000110] - - [ 111, 111, FCC00000111] - - [ 112, 112, FCC00000112] - - [ 113, 113, FCC00000113] - - [ 114, 114, FCC00000114] - - [ 115, 115, FCC00000115] - - [ 116, 116, FCC00000116] - - [ 117, 117, FCC00000117] - - [ 118, 118, FCC00000118] - - [ 119, 119, FCC00000119] - - [ 120, 120, FCC00000120] - - [ 121, 121, FCC00000121] - - [ 122, 122, FCC00000122] - - [ 123, 123, FCC00000123] - - [ 124, 124, FCC00000124] - - [ 125, 125, FCC00000125] - - [ 126, 126, FCC00000126] - - [ 127, 127, FCC00000127] diff --git a/old_tests/simulate-chime/recv.yaml b/old_tests/simulate-chime/recv.yaml deleted file mode 100644 index 4a1db7af..00000000 --- a/old_tests/simulate-chime/recv.yaml +++ /dev/null @@ -1,306 +0,0 @@ -########################################## -# -# chime_science_run_recv.yaml -# -# CHIME receiver node configuration used in the mid-November 2018 run. -# This configuration turns off saving of the 26m datasets. -# -# For the N2 data it includes, 10 second calibration data, -# full triangles for 10 frequencies at 10 seconds, and stacked -# data over all frequencies. -# -# Author: Richard Shaw -# -########################################## -type: config -log_level: info -num_elements: 128 -num_local_freq: 1 -udp_packet_size: 4928 -num_data_sets: 1 -samples_per_data_set: 4 -buffer_depth: 4 -num_gpu_frames: 128 -block_size: 32 -cpu_affinity: [1,6,7,9,14,15] -num_ev: 4 - -dataset_manager: - use_dataset_broker: True - ds_broker_host: "10.0.1.3" - -vis_pool: - kotekan_metadata_pool: visMetadata - num_metadata_objects: 500 * buffer_depth - -vis_buffer: - metadata_pool: vis_pool - num_frames: buffer_depth - visbuf_10s_all: - kotekan_buffer: vis - visbuf_10s_gain: - kotekan_buffer: vis - buffer_depth: 8 # Before slow stage - visbuf_10s_flags: - kotekan_buffer: vis - buffer_depth: 8 # Before slow stage - visbuf_10s_stack: - kotekan_buffer: vis - num_prod: 8256 # Approximation to the correct size - visbuf_10s_stack_ne: - kotekan_buffer: vis - num_ev: 0 - num_prod: 8256 # Approximation to the correct size - visbuf_10s_mfreq: - kotekan_buffer: vis - visbuf_10s_stack_mfreq: - kotekan_buffer: vis - num_prod: 8256 # Approximation to the correct size - visbuf_5s_26m_ungated: - kotekan_buffer: vis - buffer_depth: 8 - num_prod: 4096 - visbuf_5s_26m_gated: - kotekan_buffer: vis - buffer_depth: 8 - num_prod: 4096 - visbuf_5s_26m_ungated_post: - kotekan_buffer: vis - buffer_depth: 8 - num_prod: 4096 - visbuf_5s_26m_gated_post: - kotekan_buffer: vis - buffer_depth: 8 - num_prod: 4096 - visbuf_10s_cal: - kotekan_buffer: vis - num_prod: 2048 - buffer_depth: 8 # Increase as this subset is produced very quickly - visbuf_10s_timing: - num_prod: 66 - kotekan_buffer: vis - buffer_depth: 8 # Increase as this subset is produced very quickly - visbuf_10s_timing_ne: - num_prod: 66 - num_ev: 0 - kotekan_buffer: vis - buffer_depth: 8 # Increase as this subset is produced very quickly - -# Subset of good frequencies to output whole for monitoring -mfreq: &mfreq [107, 344, 619, 938] -# Larger subset to be sent to map maker, spanning entire band -mapfreq: &mapfreq [ - 0, 15, 27, 40, 53, 66, 79, 92, 104, 107, - 184, 196, 209, 220, 234, 245, 259, 281, 293, 305, 318, 333, 344, - 358, 374, 386, 399, 411, 422, 433, 445, 456, 469, 481, - 494, 506, 520, 534, 546, 610, 619, 623, 671, 703, 715, 727, - 740, 752, 805, 824, 837, 852, 863, 886, 903, 917, 928, - 938, 950, 962, 975, 986, 1000, 1011 - ] - -# Updatable config blocks -updatable_config: - flagging: - kotekan_update_endpoint: "json" - start_time: 1535048997. - tag: "initial_flags" - bad_inputs: [ ] - gains: - kotekan_update_endpoint: "json" - start_time: 1541039597. - tag: "init_gain" - 26m_gated: - kotekan_update_endpoint: "json" - enabled: false - 26m_ungated: - kotekan_update_endpoint: "json" - enabled: false - -# Kotekan stages -buffer_recv: - n2: - kotekan_stage: bufferRecv - buf: visbuf_10s_all - listen_port: 11024 - cpu_affinity: [0, 8] - num_threads: 2 - 26m_ungated: - kotekan_stage: bufferRecv - buf: visbuf_5s_26m_ungated - listen_port: 11025 - 26m_gated: - kotekan_stage: bufferRecv - buf: visbuf_5s_26m_gated - listen_port: 11026 - -switch_26m_ungated: - kotekan_stage: bufferSwitch - in_bufs: - - enabled: visbuf_5s_26m_ungated - out_buf: visbuf_5s_26m_ungated_post - updatable_config: "/updatable_config/26m_ungated" - -switch_26m_gated: - kotekan_stage: bufferSwitch - in_bufs: - - enabled: visbuf_5s_26m_gated - out_buf: visbuf_5s_26m_gated_post - updatable_config: "/updatable_config/26m_gated" - -apply_flags: - kotekan_stage: receiveFlags - in_buf: visbuf_10s_all - out_buf: visbuf_10s_flags - updatable_config: "/updatable_config/flagging" - -vis_debug: - n2: - kotekan_stage: visDebug - in_buf: visbuf_10s_flags - 26m_ungated: - kotekan_stage: visDebug - in_buf: visbuf_5s_26m_ungated_post - 26m_gated: - kotekan_stage: visDebug - in_buf: visbuf_5s_26m_gated_post - -count_check: - n2: - kotekan_stage: countCheck - in_buf: visbuf_10s_flags - 26m_ungated: - kotekan_stage: countCheck - in_buf: visbuf_5s_26m_ungated - 26m_gated: - kotekan_stage: countCheck - in_buf: visbuf_5s_26m_gated - -apply_gains: - cpu_affinity: [4, 5] - num_threads: 2 - kotekan_stage: applyGains - in_buf: visbuf_10s_flags - out_buf: visbuf_10s_gain - gains_dir: "/test/data" - updatable_config: "/updatable_config/gains" - -stacking: - kotekan_stage: baselineCompression - in_buf: visbuf_10s_gain - out_buf: visbuf_10s_stack - stack_type: chime_in_cyl - exclude_inputs: [ - 46, 142, 688, 944, 960, 1058, 1166, 1225, 1314, 1521, 1543, 2032, 2034 - ] - num_threads: 2 - cpu_affinity: [2, 3] - -buffer_status: - kotekan_stage: bufferStatus - print_status: false - -# Generate the calibration stream -cal_subset: - kotekan_stage: prodSubset - in_buf: visbuf_10s_flags - out_buf: visbuf_10s_cal - prod_subset_type: autos - -timing_subset: - kotekan_stage: prodSubset - in_buf: visbuf_10s_flags - out_buf: visbuf_10s_timing - prod_subset_type: only_inputs - input_list: [46, 142, 688, 944, 960, 1058, 1166, 1314, 1543, 2032, 2034] - -# Generate the monitoring freq full N^2 stream -mfreq_subset: - n2: - kotekan_stage: freqSubset - in_buf: visbuf_10s_flags - out_buf: visbuf_10s_mfreq - subset_list: *mfreq - stack: - kotekan_stage: freqSubset - in_buf: visbuf_10s_stack - out_buf: visbuf_10s_stack_mfreq - subset_list: *mapfreq - -buffer_writers: - file_length: 256 - root_path: /mnt/recv1/buffer/ - write_ev: True - file_base: buffer - - cal: - kotekan_stage: visCalWriter - in_buf: visbuf_10s_cal - instrument_name: chimecal - file_length: 256 - dir_name: cal - - timing: - kotekan_stage: visCalWriter - in_buf: visbuf_10s_timing - instrument_name: chimetiming - dir_name: timing - -remove_ev: - - chimestack: - kotekan_stage: removeEv - in_buf: visbuf_10s_stack - out_buf: visbuf_10s_stack_ne - - timing: - kotekan_stage: removeEv - in_buf: visbuf_10s_timing - out_buf: visbuf_10s_timing_ne - - -archive_writers: - - file_length: 512 - file_type: raw - root_path: /data/untransposed/ - - n2_mf: - kotekan_stage: visWriter - in_buf: visbuf_10s_mfreq - instrument_name: chimeN2 - - chimestack: - kotekan_stage: visWriter - in_buf: visbuf_10s_stack_ne - instrument_name: chimestack - - 26m: - kotekan_stage: visWriter - in_buf: visbuf_5s_26m_ungated_post - instrument_name: chime26m - - 26mgated: - kotekan_stage: visWriter - in_buf: visbuf_5s_26m_gated_post - instrument_name: chime26mgated - - cal: - kotekan_stage: visWriter - in_buf: visbuf_10s_cal - instrument_name: chimecal - file_length: 256 - - timing: - kotekan_stage: visWriter - in_buf: visbuf_10s_timing_ne - instrument_name: chimetiming - - -# Transmit part of the stack to the recv1 for testing -buffer_send: - server_ip: 10.1.50.11 - stack_mfreq: - kotekan_stage: bufferSend - buf: visbuf_10s_stack_mfreq - server_port: 14096 - diff --git a/old_tests/simulate-chime/run_test.sh b/old_tests/simulate-chime/run_test.sh deleted file mode 100755 index ad74c983..00000000 --- a/old_tests/simulate-chime/run_test.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -# Build kotekan and dependencies -echo "===== Checking highfive repository. =============" -if cd highfive; then git pull && cd ..; else git clone --single-branch --branch extensible-datasets https://github.com/jrs65/highfive.git; fi - -echo "===== Checking kotekan repository. ==============" -if cd kotekan; then git pull; else git clone --single-branch --branch rn/stdout https://github.com/kotekan/kotekan.git && cd kotekan; fi -git status -if cd build; then cd ..; else mkdir build; fi -cd .. - -echo "===== Building docker images. ===================" -export DOCKER_CLIENT_TIMEOUT=120 -export COMPOSE_HTTP_TIMEOUT=120 -docker-compose -f docker-compose.yaml build - -echo "===== Starting docker images. ===================" -docker-compose -f docker-compose.yaml up --scale gpu-cn01=10 -d - -echo "===== Starting fake GPS server. =================" -while : ; do ( echo -ne "HTTP/1.0 200 OK\r\nContent-Length: $(wc -c