diff --git a/src/arguments.c b/src/arguments.c index 34efb8d..b6c9171 100644 --- a/src/arguments.c +++ b/src/arguments.c @@ -10,6 +10,7 @@ enum TdoError tdo_arguments_parse(struct TdoArguments *args, int argc, char **ar enum TdoError result = TDO_ERROR_OK; *args = (struct TdoArguments) { .processes = 1, + .time_limit = 5.0, .single_test = NULL, .test_file = NULL, .output = NULL, @@ -78,6 +79,30 @@ enum TdoError tdo_arguments_parse(struct TdoArguments *args, int argc, char **ar argc -= 1; argv += 1; args->internal_status = argv[0]; } + } else if (strncmp(s, "--timeout", 10) == 0) { + if (argc <= 1) { + fprintf(stderr, "Missing timeout argument\n"); + result = TDO_ERROR_ARG_PARSE; + } else { + argc -= 1; argv += 1; + char const *timeout_str = argv[0]; + + errno = 0; + char *err; + float timeout = strtof(timeout_str, &err); + if (errno) { + perror("Could not parse amount of processes"); + result = TDO_ERROR_ARG_PARSE; + } else if (*err != '\0') { + fprintf(stderr, "Could not parse amount of threads: '%s'\n", timeout_str); + result = TDO_ERROR_ARG_PARSE; + } else if (timeout <= 0) { + fprintf(stderr, "Amount of processes must be strictly positive, got %f\n", timeout); + result = TDO_ERROR_ARG_PARSE; + } else { + args->time_limit = timeout; + } + } } else { fprintf(stderr, "Unrecognized argument: '%s'\n", s); result = TDO_ERROR_ARG_PARSE; diff --git a/src/arguments.h b/src/arguments.h index 4303bde..2bc7e80 100644 --- a/src/arguments.h +++ b/src/arguments.h @@ -8,6 +8,7 @@ struct TdoArguments { size_t processes; + float time_limit; char const *single_test; char const *test_file; char const *output; diff --git a/src/platform/run_posix.c b/src/platform/run_posix.c index 7a1a792..2838dd9 100644 --- a/src/platform/run_posix.c +++ b/src/platform/run_posix.c @@ -10,6 +10,7 @@ #include #include #include +#include struct TdoRun { struct TdoTest *test; @@ -187,7 +188,7 @@ void tdo_run_poll_exit(struct TdoRun *run, struct TdoRunStatus *status, struct T if (status->finished > 0) fprintf(output, ","); if (out_err == TDO_ERROR_OK && err_err == TDO_ERROR_OK && status_err == TDO_ERROR_OK) { - tdo_run_report_status(run, arena, output, return_status, duration); + tdo_run_report_status(run, arena, output, return_status, duration, false); } else { tdo_run_report_error(*run->test, output, NULL, "could not read output", duration); } @@ -262,10 +263,31 @@ void tdo_run_poll_event(struct TdoRunStatus *status, struct TdoArena *arena, str } } + struct timespec end_time = tdo_time_get(); + for (size_t i = 0; i < args.processes; i++) { struct TdoRun *run = &status->runs[i]; if (run->active) { tdo_run_poll_exit(run, status, arena, output); + if (run->active) { + double duration = ( + (double)(end_time.tv_sec - run->start_time.tv_sec) + + (double)(end_time.tv_nsec - run->start_time.tv_nsec) * 1e-9 + ); + if (duration > args.time_limit) { + // timeout + if (status->finished > 0) fprintf(output, ","); + tdo_run_report_status(run, arena, output, 0, duration, true); + + kill(run->pid, SIGKILL); + run->active = false; + close(run->out.fd); + close(run->err.fd); + close(run->status.fd); + status->running -= 1; + status->finished += 1; + } + } } } } diff --git a/src/platform/run_windows.c b/src/platform/run_windows.c index 8fd4345..54d30f0 100644 --- a/src/platform/run_windows.c +++ b/src/platform/run_windows.c @@ -28,6 +28,7 @@ struct TdoRun { struct TdoString out_name; struct TdoString err_name; struct TdoString status_name; + bool timed_out; bool active; struct TdoOverlap out_ov; struct TdoOverlap err_ov; @@ -255,6 +256,7 @@ void tdo_run_start_new(struct TdoRunStatus *status, struct TdoArena *arena, stru ResumeThread(pi.hThread); CloseHandle(pi.hThread); + run->timed_out = false; run->active = true; run->start_time = start_time; run->pid = pi.dwProcessId; @@ -293,7 +295,7 @@ void tdo_run_maybe_report_exit(struct TdoArena *arena, struct TdoRun *run, struc double duration = (double)(end_time.QuadPart - run->start_time.QuadPart) / status->clock_frequency.QuadPart; if (status->finished > 0) fprintf(output, ","); - tdo_run_report_status(run, arena, output, run->exit_code, duration); + tdo_run_report_status(run, arena, output, run->exit_code, duration, run->timed_out); run->active = false; status->running -= 1; @@ -301,7 +303,7 @@ void tdo_run_maybe_report_exit(struct TdoArena *arena, struct TdoRun *run, struc } void tdo_run_handle_exit(struct TdoArena *arena, struct TdoRun *run, struct TdoRunStatus *status, FILE *output, DWORD pid, DWORD msg) { - if (msg != JOB_OBJECT_MSG_EXIT_PROCESS) return; + if (msg != JOB_OBJECT_MSG_END_OF_PROCESS_TIME && msg != JOB_OBJECT_MSG_EXIT_PROCESS) return; if (run == NULL) { fprintf(stderr, "Process with unknown PID exited\n"); @@ -309,6 +311,14 @@ void tdo_run_handle_exit(struct TdoArena *arena, struct TdoRun *run, struct TdoR abort(); } + if (msg == JOB_OBJECT_MSG_END_OF_PROCESS_TIME) { + run->timed_out = true; + CloseHandle(run->process_handle); + run->process_handle = NULL; + run->exit_code = 0; + return; + } + DWORD return_status; if (!GetExitCodeProcess(run->process_handle, &return_status)) { fprintf(stderr, "Get exit code failed somehow... TODO: graceful exit\n"); @@ -556,7 +566,8 @@ enum TdoError tdo_run_status_init(struct TdoRunStatus *status, struct TdoArena * } JOBOBJECT_EXTENDED_LIMIT_INFORMATION jeli = {0}; - jeli.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + jeli.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE | JOB_OBJECT_LIMIT_PROCESS_TIME; + jeli.BasicLimitInformation.PerProcessUserTimeLimit.QuadPart = (LONGLONG)(args.time_limit * 1e7); if (!SetInformationJobObject(status->job, JobObjectExtendedLimitInformation, &jeli, sizeof(jeli))) { result = TDO_ERROR_OS; diff --git a/src/run.c b/src/run.c index 63f2cc7..e32d43f 100644 --- a/src/run.c +++ b/src/run.c @@ -44,7 +44,7 @@ void tdo_log_dump(struct TdoLog log, FILE *file, char const *name) { fprintf(file, "\""); } -void tdo_run_report_exit(struct TdoRun *run, FILE *file, char const *step, TdoProcessStatus status, double duration) { +void tdo_run_report_exit(struct TdoRun *run, FILE *file, char const *step, TdoProcessStatus status, double duration, bool timed_out) { fprintf(file, "\n"); fprintf(file, "\t{\n"); @@ -59,7 +59,9 @@ void tdo_run_report_exit(struct TdoRun *run, FILE *file, char const *step, TdoPr fprintf(file, "\t\t\"duration\": %lf,\n", duration); fprintf(file, "\t\t\"status\": \""); - if (tdo_process_status_is_exit(status)) { + if (timed_out) { + fprintf(file, "timeout"); + } else if (tdo_process_status_is_exit(status)) { if (step[0] == 'f') { fprintf(file, "complete"); } else { @@ -72,15 +74,17 @@ void tdo_run_report_exit(struct TdoRun *run, FILE *file, char const *step, TdoPr } fprintf(file, "\""); - if (tdo_process_status_is_exit(status) && step[0] != 'f') { - fprintf(file, ",\n\t\t\"exit\": " TDO_PROCESS_CODE_FORMAT, tdo_process_code_exit(status)); - } else if (tdo_process_status_is_signal(status)) { - fprintf(file, ",\n\t\t\"signal\": " TDO_PROCESS_CODE_FORMAT, tdo_process_code_signal(status)); - } else if (tdo_process_status_is_stop(status)) { - fprintf(file, ",\n\t\t\"stop\": " TDO_PROCESS_CODE_FORMAT, tdo_process_code_stop(status)); + if (!timed_out) { + if (tdo_process_status_is_exit(status) && step[0] != 'f') { + fprintf(file, ",\n\t\t\"exit\": " TDO_PROCESS_CODE_FORMAT, tdo_process_code_exit(status)); + } else if (tdo_process_status_is_signal(status)) { + fprintf(file, ",\n\t\t\"signal\": " TDO_PROCESS_CODE_FORMAT, tdo_process_code_signal(status)); + } else if (tdo_process_status_is_stop(status)) { + fprintf(file, ",\n\t\t\"stop\": " TDO_PROCESS_CODE_FORMAT, tdo_process_code_stop(status)); + } } - if (step[0] != 'f') { + if (timed_out || step[0] != 'f') { fprintf(file, ",\n\t\t\"step\": \""); tdo_json_escaped(file, (struct TdoString) { .length=strlen(step), .bytes=(char*)step }); fprintf(file, "\""); @@ -123,6 +127,54 @@ void tdo_run_report_error(struct TdoTest test, FILE *file, char const *step, cha fprintf(file, "\t}"); } +void tdo_run_report_timeout(struct TdoRun *run, FILE *file, char const *step, TdoProcessStatus status, double duration) { + fprintf(file, "\n"); + fprintf(file, "\t{\n"); + + fprintf(file, "\t\t\"file\": \""); + tdo_json_escaped(file, run->test->symbol.file->name); + fprintf(file, "\",\n"); + + fprintf(file, "\t\t\"name\": \""); + tdo_json_escaped(file, run->test->symbol.name); + fprintf(file, "\",\n"); + + fprintf(file, "\t\t\"duration\": %lf,\n", duration); + + fprintf(file, "\t\t\"status\": \""); + if (tdo_process_status_is_exit(status)) { + if (step[0] == 'f') { + fprintf(file, "complete"); + } else { + fprintf(file, "exit"); + } + } else if (tdo_process_status_is_signal(status)) { + fprintf(file, "signal"); + } else if (tdo_process_status_is_stop(status)) { + fprintf(file, "stop"); + } + fprintf(file, "\""); + + if (tdo_process_status_is_exit(status) && step[0] != 'f') { + fprintf(file, ",\n\t\t\"exit\": " TDO_PROCESS_CODE_FORMAT, tdo_process_code_exit(status)); + } else if (tdo_process_status_is_signal(status)) { + fprintf(file, ",\n\t\t\"signal\": " TDO_PROCESS_CODE_FORMAT, tdo_process_code_signal(status)); + } else if (tdo_process_status_is_stop(status)) { + fprintf(file, ",\n\t\t\"stop\": " TDO_PROCESS_CODE_FORMAT, tdo_process_code_stop(status)); + } + + if (step[0] != 'f') { + fprintf(file, ",\n\t\t\"step\": \""); + tdo_json_escaped(file, (struct TdoString) { .length=strlen(step), .bytes=(char*)step }); + fprintf(file, "\""); + } + + tdo_log_dump(run->out, file, "stdout"); + tdo_log_dump(run->err, file, "stderr"); + + fprintf(file, "\n\t}"); +} + enum TdoError tdo_string_previous_line(struct TdoString *line, struct TdoString string, size_t index) { if (string.bytes == NULL || string.length == 0) return TDO_ERROR_EOF; if (string.bytes[index] != '\n') return TDO_ERROR_NEWLINE; @@ -195,7 +247,7 @@ enum TdoError tdo_run_report_assemble_step(struct TdoString *step, struct TdoAre return TDO_ERROR_OK; } -void tdo_run_report_status(struct TdoRun *run, struct TdoArena *arena, FILE *file, int status, double duration) { +void tdo_run_report_status(struct TdoRun *run, struct TdoArena *arena, FILE *file, int status, double duration, bool timed_out) { struct TdoArenaState state = tdo_arena_state_get(arena); struct TdoString log_status = run->status.data; @@ -302,7 +354,7 @@ void tdo_run_report_status(struct TdoRun *run, struct TdoArena *arena, FILE *fil goto done; } - tdo_run_report_exit(run, file, step.bytes, status, duration); + tdo_run_report_exit(run, file, step.bytes, status, duration, timed_out); } else if (strncmp(last_line.bytes, "test", 4) == 0) { struct TdoString step; enum TdoError err_step = tdo_run_report_assemble_step(&step, arena, last_line, run->test->symbol); @@ -312,10 +364,10 @@ void tdo_run_report_status(struct TdoRun *run, struct TdoArena *arena, FILE *fil } // unexpected exit while running test - tdo_run_report_exit(run, file, step.bytes, status, duration); + tdo_run_report_exit(run, file, step.bytes, status, duration, timed_out); } else if (strncmp(last_line.bytes, "finished", 8) == 0) { // test finished normally - tdo_run_report_exit(run, file, last_line.bytes, status, duration); + tdo_run_report_exit(run, file, last_line.bytes, status, duration, timed_out); } else { // unknown status tdo_run_report_error(*run->test, file, NULL, "unknown error", duration); diff --git a/src/run.h b/src/run.h index 11ec878..48abe4d 100644 --- a/src/run.h +++ b/src/run.h @@ -12,8 +12,8 @@ enum TdoError tdo_parse_size_t(size_t *number, char const *string); enum TdoError tdo_run_report_assemble_step(struct TdoString *step, struct TdoArena *arena, struct TdoString step_name, struct TdoSymbol symbol); -void tdo_run_report_status(struct TdoRun *run, struct TdoArena *arena, FILE *file, int status, double duration); -void tdo_run_report_exit(struct TdoRun *run, FILE *file, char const *step, TdoProcessStatus status, double duration); +void tdo_run_report_status(struct TdoRun *run, struct TdoArena *arena, FILE *file, int status, double duration, bool timed_out); +void tdo_run_report_exit(struct TdoRun *run, FILE *file, char const *step, TdoProcessStatus status, double duration, bool timed_out); void tdo_run_report_error(struct TdoTest test, FILE *file, char const *step, char const *error, double duration); void tdo_status_error(FILE *file, char const *fmt, ...); diff --git a/test/conftest.py b/test/conftest.py index d7c2263..b390407 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -318,6 +318,11 @@ class ResultStop(ResultDone): stop: int +@dataclasses.dataclass +class ResultTimeout(ResultDone): + step: Step + + class Error(enum.Enum): ok = 0 unknown = -1 @@ -408,6 +413,9 @@ def run(tests: str, executable: Optional[str] = None, args: Optional[List[Any]] elif status == 'stop': keys = {'file', 'name', 'duration', 'status', 'stop', 'stdout', 'stderr', 'step'} result_type = ResultStop + elif status == 'timeout': + keys = {'file', 'name', 'duration', 'status', 'stdout', 'stderr', 'step'} + result_type = ResultTimeout else: raise ValueError(f'Invalid status: "{status}"') diff --git a/test/library.c b/test/library.c index 3bdb3db..260d386 100644 --- a/test/library.c +++ b/test/library.c @@ -29,3 +29,20 @@ EXPORT void test_early_exit(void) { EXPORT void test_aborts(void) { abort(); } + +EXPORT void test_print_forever(void) { + while (1) { + fprintf(stdout, "I am printing forever!\n"); + } +} + +EXPORT void test_timeout(void) { + fprintf(stdout, "Some output\n"); fflush(stdout); + fprintf(stderr, "Some error\n"); fflush(stderr); + + volatile unsigned int x = 0; + while (1) { + x++; + } +} + diff --git a/test/test_runner.py b/test/test_runner.py index 91bc00f..a61f7ab 100644 --- a/test/test_runner.py +++ b/test/test_runner.py @@ -1,5 +1,5 @@ from typing import Any -from conftest import ResultComplete, ResultError, ResultExit, ResultSignal, StepFixtureAfter, StepFixtureBefore, StepTest, RunTests, approx +from conftest import ResultComplete, ResultError, ResultExit, ResultSignal, ResultTimeout, StepFixtureAfter, StepFixtureBefore, StepTest, RunTests, approx def test_success(library: str, run_tests: RunTests): @@ -68,6 +68,38 @@ def test_early_exit(library: str, run_tests: RunTests): )] +def test_timeout(library: str, run_tests: RunTests): + result, _ = run_tests(f""" + test::{library}::test_success + test::{library}::test_timeout + test::{library}::test_success + """, args=['--timeout', 0.1]) + assert result == [ + ResultComplete( + file=library, + name='test_success', + duration=approx(0.0, abs=100.0), + stdout='', + stderr='', + ), + ResultTimeout( + file=library, + name='test_timeout', + duration=approx(0.0, abs=100.0), + step=StepTest(file=library, name='test_timeout'), + stdout='Some output\n', + stderr='Some error\n', + ), + ResultComplete( + file=library, + name='test_success', + duration=approx(0.0, abs=100.0), + stdout='', + stderr='', + ), + ] + + def test_aborts(library: str, run_tests: RunTests): result, _ = run_tests(f'test::{library}::test_aborts') assert result == [ResultSignal(