From 157a5858bd574bdfa61f235c8203ecad93195a17 Mon Sep 17 00:00:00 2001 From: Joeri Exelmans Date: Fri, 7 Oct 2022 12:57:28 +0200 Subject: [PATCH 1/6] Run Python process as child process of NodeJS. Graceful and forced shutdown of both processes. --- httpwsd.js | 72 +++++++++++++++++++++++++++++++++++++++++---- mt/main.py | 2 +- run_AToMPM_local.sh | 42 -------------------------- 3 files changed, 68 insertions(+), 48 deletions(-) delete mode 100755 run_AToMPM_local.sh diff --git a/httpwsd.js b/httpwsd.js index 01c7e663..c429c171 100644 --- a/httpwsd.js +++ b/httpwsd.js @@ -4,6 +4,7 @@ */ /*********************************** IMPORTS **********************************/ +const process = require('node:process'); const _cp = require('child_process'), _fs = require('fs'), _http = require('http'), @@ -603,10 +604,71 @@ let httpserver = _http.createServer( session_manager.init_session_manager(httpserver); -let port = 8124; -httpserver.listen(port); +// Run Python transformation engine as a child process: +const childProcess = require('node:child_process'); +const pythonProcess = childProcess.spawn('python', ['mt/main.py']); +pythonProcess.on('exit', (code, signal) => { + logger.info("Model Transformation Server exited " + + ((code===null) ? ("by signal " + signal) : ("with code " + code.toString()))); +}); +const transformStream = (readableStream, writableStream, colorCode) => { + readableStream.on('data', chunk => { + writableStream.write("\x1b["+colorCode+"m"); // set color + writableStream.write(chunk); + writableStream.write("\x1b[0m"); // reset color + }); +}; +// output of Python process is interleaved with output of this process: +transformStream(pythonProcess.stdout, process.stdout, "33"); // yellow +transformStream(pythonProcess.stderr, process.stderr, "91"); // red + + +// Only start the HTTP server after the Transformation Server has started. +// When the Python process has written the following string to stdout, we know the transformation server has started: +const expectedString = Buffer.from("Started Model Transformation Server\n"); +let accumulatedOutput = Buffer.alloc(0); +function pythonStartedListener(chunk) { + accumulatedOutput = Buffer.concat([accumulatedOutput, chunk]); + + if (accumulatedOutput.length >= expectedString.length) { + if (accumulatedOutput.subarray(0, expectedString.length).equals(expectedString)) { + // No need to keep accumulating pythonProcess' stdout: + pythonProcess.stdout.removeListener('data', pythonStartedListener); + + httpserver.on('close', () => { + pythonProcess.kill('SIGTERM'); // cleanly exit Python process AFTER http server has shut down. + }) + + let port = 8124; + httpserver.listen(port); + + logger.info(`HTTP server running on: http://localhost:${port}/atompm`); + logger.info("```mermaid"); + logger.info("sequenceDiagram"); + } + } +} +const waitUntilExpectedString = pythonProcess.stdout.on('data', pythonStartedListener); + -logger.info("AToMPM listening on port: " + port); -logger.info("```mermaid"); -logger.info("sequenceDiagram"); +function gracefulShutdown() { + logger.info("Gracefully shutting down..."); + httpserver.close(); +} +process.once('SIGINT', () => { + // The Python process belongs to the same process group, and will also receive a SIGINT. + // This is not necessary, because we send a SIGTERM to the Python process anyway, but it won't cause us trouble. + process.once('SIGINT', () => { + process.exit(1); + }); + logger.info(""); + logger.info("Received SIGINT. Send another SIGINT to force shutdown."); + gracefulShutdown(); +}); +process.once('SIGTERM', gracefulShutdown); + +process.on('exit', code => { + // In case of a forced exit (e.g. uncaught exception), the Python process may still be running, so we force-kill it: + pythonProcess.kill('SIGKILL'); +}); diff --git a/mt/main.py b/mt/main.py index 75a3f7b9..be547d4c 100644 --- a/mt/main.py +++ b/mt/main.py @@ -12,9 +12,9 @@ def main() : logging.basicConfig(format='%(levelname)s - %(message)s', level=logging.INFO) - print("Starting Model Transformation Server... ") httpd = HTTPServerThread() httpd.start() + print("Started Model Transformation Server") if __name__ == "__main__" : main() diff --git a/run_AToMPM_local.sh b/run_AToMPM_local.sh deleted file mode 100755 index fb039c2e..00000000 --- a/run_AToMPM_local.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -# Author: Yentl Van Tendeloo and Bentley James Oakes - -set -e - -echo "Running AToMPM script..." -echo "Starting node..." -node httpwsd.js & -serverpid=$! -sleep 3 - -echo "Starting Python model transformation engine..." -python mt/main.py& -mtpid=$! -sleep 1 - -echo "AToMPM now running. Opening browser..." -set +e - - -echo "Trying to run Chromium..." -chromium-browser http://localhost:8124/atompm - -ret=$? -if [ $ret -ne 0 ]; then - echo "Chromium not installed. Trying Google Chrome..." - google-chrome-stable http://localhost:8124/atompm -fi - -ret=$? -if [ $ret -ne 0 ]; then - echo "Google Chrome not installed. Trying Firefox..." - firefox http://localhost:8124/atompm -fi - -ret=$? -if [ $ret = 0 ]; then - echo "Stopping AToMPM" - kill $serverpid - kill $mtpid - echo "Finished!" -fi From 1c9bf46d25f28f3331b71188e0312fa07e08a488 Mon Sep 17 00:00:00 2001 From: Joeri Exelmans Date: Fri, 7 Oct 2022 13:13:01 +0200 Subject: [PATCH 2/6] Add option to run NodeJS without Python child process. --- httpwsd.js | 136 +++++++++++++++++++++++++++++------------------------ 1 file changed, 74 insertions(+), 62 deletions(-) diff --git a/httpwsd.js b/httpwsd.js index c429c171..7c37c494 100644 --- a/httpwsd.js +++ b/httpwsd.js @@ -5,7 +5,7 @@ /*********************************** IMPORTS **********************************/ const process = require('node:process'); -const _cp = require('child_process'), +const _cp = require('node:child_process'), _fs = require('fs'), _http = require('http'), _url = require('url'), @@ -15,6 +15,11 @@ const _cp = require('child_process'), _utils = require('./utils'); const session_manager = require("./session_manager"); +// Command line flag: +const runWithPythonChildProcess = !(process.argv.slice(2).includes("--without-child")); // default is true + +const port = 8124; + /** Wrapper function to log HTTP messages from the server **/ function __respond(response, statusCode, reason, data, headers) @@ -604,71 +609,78 @@ let httpserver = _http.createServer( session_manager.init_session_manager(httpserver); +function startServer() { + httpserver.listen(port); + logger.info(`Server listening on: http://localhost:${port}/atompm`); + logger.info("```mermaid"); + logger.info("sequenceDiagram"); + + function gracefulShutdown() { + logger.info("Gracefully shutting down..."); + httpserver.close(); + } + process.once('SIGINT', () => { + // The Python process belongs to the same process group, and will also receive a SIGINT. + // This is not necessary, because we send a SIGTERM to the Python process anyway, but it won't cause us trouble. + + // The next SIGINT will cause a forced exit: + process.once('SIGINT', () => { + process.exit(1); + }); + logger.info(""); + logger.info("Received SIGINT. Send another SIGINT to force shutdown."); + gracefulShutdown(); + }); + process.once('SIGTERM', gracefulShutdown); +} + // Run Python transformation engine as a child process: -const childProcess = require('node:child_process'); -const pythonProcess = childProcess.spawn('python', ['mt/main.py']); -pythonProcess.on('exit', (code, signal) => { - logger.info("Model Transformation Server exited " - + ((code===null) ? ("by signal " + signal) : ("with code " + code.toString()))); -}); -const transformStream = (readableStream, writableStream, colorCode) => { - readableStream.on('data', chunk => { - writableStream.write("\x1b["+colorCode+"m"); // set color - writableStream.write(chunk); - writableStream.write("\x1b[0m"); // reset color +if (runWithPythonChildProcess) { + const pythonProcess = _cp.spawn('python', ['mt/main.py']); + pythonProcess.on('exit', (code, signal) => { + logger.info("Model Transformation Server exited " + + ((code===null) ? ("by signal " + signal) : ("with code " + code.toString()))); }); -}; -// output of Python process is interleaved with output of this process: -transformStream(pythonProcess.stdout, process.stdout, "33"); // yellow -transformStream(pythonProcess.stderr, process.stderr, "91"); // red - - -// Only start the HTTP server after the Transformation Server has started. -// When the Python process has written the following string to stdout, we know the transformation server has started: -const expectedString = Buffer.from("Started Model Transformation Server\n"); -let accumulatedOutput = Buffer.alloc(0); -function pythonStartedListener(chunk) { - accumulatedOutput = Buffer.concat([accumulatedOutput, chunk]); - - if (accumulatedOutput.length >= expectedString.length) { - if (accumulatedOutput.subarray(0, expectedString.length).equals(expectedString)) { - // No need to keep accumulating pythonProcess' stdout: - pythonProcess.stdout.removeListener('data', pythonStartedListener); - - httpserver.on('close', () => { - pythonProcess.kill('SIGTERM'); // cleanly exit Python process AFTER http server has shut down. - }) - - let port = 8124; - httpserver.listen(port); - - logger.info(`HTTP server running on: http://localhost:${port}/atompm`); - logger.info("```mermaid"); - logger.info("sequenceDiagram"); + const coloredStream = (readableStream, writableStream, colorCode) => { + readableStream.on('data', chunk => { + writableStream.write("\x1b["+colorCode+"m"); // set color + writableStream.write(chunk); + writableStream.write("\x1b[0m"); // reset color + }); + }; + // output of Python process is interleaved with output of this process: + coloredStream(pythonProcess.stdout, process.stdout, "33"); // yellow + coloredStream(pythonProcess.stderr, process.stderr, "91"); // red + + // Only start the HTTP server after the Transformation Server has started. + // When the Python process has written the following string to stdout, we know the transformation server has started: + const expectedString = Buffer.from("Started Model Transformation Server\n"); + let accumulatedOutput = Buffer.alloc(0); + function pythonStartedListener(chunk) { + accumulatedOutput = Buffer.concat([accumulatedOutput, chunk]); + + if (accumulatedOutput.length >= expectedString.length) { + if (accumulatedOutput.subarray(0, expectedString.length).equals(expectedString)) { + // No need to keep accumulating pythonProcess' stdout: + pythonProcess.stdout.removeListener('data', pythonStartedListener); + + httpserver.on('close', () => { + pythonProcess.kill('SIGTERM'); // cleanly exit Python process AFTER http server has shut down. + }); + + startServer(); + } } } -} -const waitUntilExpectedString = pythonProcess.stdout.on('data', pythonStartedListener); + const waitUntilExpectedString = pythonProcess.stdout.on('data', pythonStartedListener); - -function gracefulShutdown() { - logger.info("Gracefully shutting down..."); - httpserver.close(); -} - -process.once('SIGINT', () => { - // The Python process belongs to the same process group, and will also receive a SIGINT. - // This is not necessary, because we send a SIGTERM to the Python process anyway, but it won't cause us trouble. - process.once('SIGINT', () => { - process.exit(1); + // In case of a forced exit (e.g. uncaught exception), the Python process may still be running, so we force-kill it: + process.on('exit', code => { + pythonProcess.kill('SIGKILL'); }); - logger.info(""); - logger.info("Received SIGINT. Send another SIGINT to force shutdown."); - gracefulShutdown(); -}); -process.once('SIGTERM', gracefulShutdown); +} +else { + // Run without Python as a child process + startServer(); +} -process.on('exit', code => { - // In case of a forced exit (e.g. uncaught exception), the Python process may still be running, so we force-kill it: - pythonProcess.kill('SIGKILL'); -}); From c92c3eb87c1414220c3196e1b3b39ca90c0f9b5f Mon Sep 17 00:00:00 2001 From: Joeri Exelmans Date: Thu, 3 Nov 2022 15:02:52 +0100 Subject: [PATCH 3/6] Make Python child process opt-in instead of opt-out. --- httpwsd.js | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/httpwsd.js b/httpwsd.js index 7c37c494..292ab8b5 100644 --- a/httpwsd.js +++ b/httpwsd.js @@ -16,7 +16,7 @@ const _cp = require('node:child_process'), const session_manager = require("./session_manager"); // Command line flag: -const runWithPythonChildProcess = !(process.argv.slice(2).includes("--without-child")); // default is true +const runWithPythonChildProcess = process.argv.slice(2).includes("--with-python"); // opt-in const port = 8124; @@ -641,6 +641,9 @@ if (runWithPythonChildProcess) { logger.info("Model Transformation Server exited " + ((code===null) ? ("by signal " + signal) : ("with code " + code.toString()))); }); + httpserver.on('close', () => { + pythonProcess.kill('SIGTERM'); // cleanly exit Python process AFTER http server has shut down. + }); const coloredStream = (readableStream, writableStream, colorCode) => { readableStream.on('data', chunk => { writableStream.write("\x1b["+colorCode+"m"); // set color @@ -664,15 +667,11 @@ if (runWithPythonChildProcess) { // No need to keep accumulating pythonProcess' stdout: pythonProcess.stdout.removeListener('data', pythonStartedListener); - httpserver.on('close', () => { - pythonProcess.kill('SIGTERM'); // cleanly exit Python process AFTER http server has shut down. - }); - startServer(); } } } - const waitUntilExpectedString = pythonProcess.stdout.on('data', pythonStartedListener); + pythonProcess.stdout.on('data', pythonStartedListener); // In case of a forced exit (e.g. uncaught exception), the Python process may still be running, so we force-kill it: process.on('exit', code => { From 1da5f471c0a823f2f95c95dc5591247698b9a62c Mon Sep 17 00:00:00 2001 From: Joeri Exelmans Date: Thu, 3 Nov 2022 15:05:55 +0100 Subject: [PATCH 4/6] Restore runner script --- run_AToMPM_local.sh | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 run_AToMPM_local.sh diff --git a/run_AToMPM_local.sh b/run_AToMPM_local.sh new file mode 100644 index 00000000..fb039c2e --- /dev/null +++ b/run_AToMPM_local.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Author: Yentl Van Tendeloo and Bentley James Oakes + +set -e + +echo "Running AToMPM script..." +echo "Starting node..." +node httpwsd.js & +serverpid=$! +sleep 3 + +echo "Starting Python model transformation engine..." +python mt/main.py& +mtpid=$! +sleep 1 + +echo "AToMPM now running. Opening browser..." +set +e + + +echo "Trying to run Chromium..." +chromium-browser http://localhost:8124/atompm + +ret=$? +if [ $ret -ne 0 ]; then + echo "Chromium not installed. Trying Google Chrome..." + google-chrome-stable http://localhost:8124/atompm +fi + +ret=$? +if [ $ret -ne 0 ]; then + echo "Google Chrome not installed. Trying Firefox..." + firefox http://localhost:8124/atompm +fi + +ret=$? +if [ $ret = 0 ]; then + echo "Stopping AToMPM" + kill $serverpid + kill $mtpid + echo "Finished!" +fi From 93e427b24587951fffbd0d6de21d395dbe4b5d07 Mon Sep 17 00:00:00 2001 From: Joeri Exelmans Date: Thu, 3 Nov 2022 15:39:04 +0100 Subject: [PATCH 5/6] Moved Python child process running/killing logic to separate module. --- httpwsd.js | 47 ++++----------------------------------------- pythonrunner.js | 51 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 43 deletions(-) create mode 100644 pythonrunner.js diff --git a/httpwsd.js b/httpwsd.js index 292ab8b5..68af6bb9 100644 --- a/httpwsd.js +++ b/httpwsd.js @@ -15,7 +15,7 @@ const _cp = require('node:child_process'), _utils = require('./utils'); const session_manager = require("./session_manager"); -// Command line flag: +// Command line flag: Start python model transformation server process as a child process of this (node) process. const runWithPythonChildProcess = process.argv.slice(2).includes("--with-python"); // opt-in const port = 8124; @@ -634,49 +634,10 @@ function startServer() { process.once('SIGTERM', gracefulShutdown); } -// Run Python transformation engine as a child process: if (runWithPythonChildProcess) { - const pythonProcess = _cp.spawn('python', ['mt/main.py']); - pythonProcess.on('exit', (code, signal) => { - logger.info("Model Transformation Server exited " - + ((code===null) ? ("by signal " + signal) : ("with code " + code.toString()))); - }); - httpserver.on('close', () => { - pythonProcess.kill('SIGTERM'); // cleanly exit Python process AFTER http server has shut down. - }); - const coloredStream = (readableStream, writableStream, colorCode) => { - readableStream.on('data', chunk => { - writableStream.write("\x1b["+colorCode+"m"); // set color - writableStream.write(chunk); - writableStream.write("\x1b[0m"); // reset color - }); - }; - // output of Python process is interleaved with output of this process: - coloredStream(pythonProcess.stdout, process.stdout, "33"); // yellow - coloredStream(pythonProcess.stderr, process.stderr, "91"); // red - - // Only start the HTTP server after the Transformation Server has started. - // When the Python process has written the following string to stdout, we know the transformation server has started: - const expectedString = Buffer.from("Started Model Transformation Server\n"); - let accumulatedOutput = Buffer.alloc(0); - function pythonStartedListener(chunk) { - accumulatedOutput = Buffer.concat([accumulatedOutput, chunk]); - - if (accumulatedOutput.length >= expectedString.length) { - if (accumulatedOutput.subarray(0, expectedString.length).equals(expectedString)) { - // No need to keep accumulating pythonProcess' stdout: - pythonProcess.stdout.removeListener('data', pythonStartedListener); - - startServer(); - } - } - } - pythonProcess.stdout.on('data', pythonStartedListener); - - // In case of a forced exit (e.g. uncaught exception), the Python process may still be running, so we force-kill it: - process.on('exit', code => { - pythonProcess.kill('SIGKILL'); - }); + const {startPythonChildProcess} = require("./pythonrunner"); + const {endPythonChildProcess} = startPythonChildProcess(startServer); + httpserver.on('close', endPythonChildProcess); } else { // Run without Python as a child process diff --git a/pythonrunner.js b/pythonrunner.js new file mode 100644 index 00000000..78e2a166 --- /dev/null +++ b/pythonrunner.js @@ -0,0 +1,51 @@ +const childProcess = require("node:child_process"); +const logger = require("./logger"); + +function startPythonChildProcess(pythonStartedCallback) { + const pythonProcess = childProcess.spawn("python", ["mt/main.py"]); + pythonProcess.on("exit", (code, signal) => { + logger.info("Model Transformation Server exited " + + ((code===null) ? ("by signal " + signal) : ("with code " + code.toString()))); + }); + const coloredStream = (readableStream, writableStream, colorCode) => { + readableStream.on("data", chunk => { + writableStream.write("\x1b["+colorCode+"m"); // set color + writableStream.write(chunk); + writableStream.write("\x1b[0m"); // reset color + }); + }; + // output of Python process is interleaved with output of this process: + coloredStream(pythonProcess.stdout, process.stdout, "33"); // yellow + coloredStream(pythonProcess.stderr, process.stderr, "91"); // red + + // Only start the HTTP server after the Transformation Server has started. + // When the Python process has written the following string to stdout, we know the transformation server has started: + const expectedString = Buffer.from("Started Model Transformation Server\n"); + let accumulatedOutput = Buffer.alloc(0); + function pythonOutputListener(chunk) { + accumulatedOutput = Buffer.concat([accumulatedOutput, chunk]); + + if (accumulatedOutput.length >= expectedString.length) { + if (accumulatedOutput.subarray(0, expectedString.length).equals(expectedString)) { + // No need to keep accumulating pythonProcess" stdout: + pythonProcess.stdout.removeListener("data", pythonOutputListener); + + pythonStartedCallback(); + } + } + } + pythonProcess.stdout.on("data", pythonOutputListener); + + // In case of a forced exit (e.g. uncaught exception), the Python process may still be running, so we force-kill it: + process.on("exit", code => { + pythonProcess.kill("SIGKILL"); + }); + + return { + endPythonChildProcess: () => { + pythonProcess.kill("SIGTERM"); // cleanly exit Python process AFTER http server has shut down. + }, + }; +} + +module.exports = { startPythonChildProcess }; \ No newline at end of file From 5f9aef3b3260815b1a45c6581fbf71de4c63e394 Mon Sep 17 00:00:00 2001 From: Joeri Exelmans Date: Thu, 3 Nov 2022 15:39:33 +0100 Subject: [PATCH 6/6] Update README with new --with-python flag --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index a2f7018b..15f11032 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ To run AToMPM on Windows, double-click on the `run.bat` script inside of the mai ### Mac or Linux 1. Execute `node httpwsd.js` in one terminal 2. Execute `python mt\main.py` in another terminal +2.1. Alternatively, adding the `--with-python` flag to the `node` command above will run the Python process as a child of the NodeJS process, making this step unnecessary. 3. Open a browser (Firefox or Chrome) and navigate to [http://localhost:8124/atompm](http://localhost:8124/atompm) * The above steps are automated by the `run_AToMPM_local.sh` script