fix: very bad memory leaks in low-memory conditions #976

Workflow file for this run

.github/workflows/notecard-binary-tests.yml at eb4c328

	name: note-c card.binary HIL tests

	on:
	pull_request:
	branches: [ master ]
	workflow_dispatch:
	inputs:
	firmware_s3_uri:
	description: 'S3 URI for Notecard firmware to sideload before running the tests. Special value "nightly" downloads the latest nightly firmware. If not given, tests will run using whatever firmware is currently installed.'
	type: string
	default: ''
	schedule:
	# Run nightly. * is a special character in YAML so you have to quote this
	# string.
	- cron: '0 8 * * *'

	permissions:
	checks: write
	contents: read
	id-token: write

	jobs:
	md5srv-test:
	uses: ./.github/workflows/md5srv-tests.yml

	notecard-binary-test:
	needs: md5srv-test
	# Skip on fork PRs, nightly builds, etc. -> Secrets (Tailscale, Notehub, ngrok) aren't available.
	if: >-
	github.event.repository.fork != true &&
	(github.event_name != 'pull_request' \|\| github.event.pull_request.head.repo.full_name == github.repository)
	runs-on: ubuntu-latest
	defaults:
	run:
	shell: bash
	env:
	FIRMWARE_S3_URI: ${{ github.event_name == 'schedule' && 'nightly' \|\| inputs.firmware_s3_uri }}
	MD5SRV_PORT: 9178
	NOTEHUB: "notehub.io"
	NOTEHUB_API: "api.notefile.net"
	NOTEHUB_PROJECT_UID: "app:458d7b93-8e19-45f8-b030-fb96d03eb1cc"
	NOTEHUB_PRODUCT_UID: "com.blues.hitl"
	NOTEHUB_ROUTE_TIMEOUT: 180
	PIO_PROJECT_DIR: ./test/hitl/card.binary
	NOTEHUB_PROXY_ROUTE_ALIAS: card.binary.${{github.run_id}}
	NOTEHUB_PROXY_ROUTE_LABEL: card.binary.proxy.${{github.run_id}}
	NOTEHUB_HTTP_ROUTE_LABEL: card.binary.http.${{github.run_id}}

	# Troubleshooting helpers
	# DELETE_NOTEHUB_ROUTES set to false to see the created routes on notehub
	DELETE_NOTEHUB_ROUTES: true
	# CREATE_NOTEHUB_ROUTES set to false to use the already created routes on notehub
	CREATE_NOTEHUB_ROUTES: true
	steps:
	- name: Connect to Tailscale
	uses: tailscale/github-action@v3
	with:
	oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}
	oauth-secret: ${{ secrets.TS_OAUTH_CLIENT_SECRET }}
	tags: tag:ci
	version: latest
	use-cache: 'true'

	# Needed so we can download Notecard firmware from S3.
	- name: Configure AWS credentials
	uses: aws-actions/configure-aws-credentials@v4
	with:
	role-to-assume: arn:aws:iam::163050375163:role/S3ReadAccessForHil_LabGitHubActions
	aws-region: us-east-1
	mask-aws-account-id: 'false'

	# Needed for asyncio.TaskGroup, which the card_client code uses.
	- name: Set up Python 3.11
	uses: actions/setup-python@v4
	with:
	python-version: '3.11'

	- name: Checkout note-c repo
	uses: actions/checkout@v4

	- name: Generate MD5 Server Token
	run: \|
	[ -n "$MD5SRV_TOKEN" ] \|\| echo "MD5SRV_TOKEN=`uuidgen`" >> $GITHUB_ENV

	# gdb-multiarch: We used gdb to remotely flash the test firmware onto the
	# Swan attached to the Notestation. Apparently "regular" gdb (i.e.
	# installed via apt install gdb) can't cope with the fact that the target
	# is ARM.
	# ngrok: Used to get a public IP for the MD5 server.
	# jq: Used for JSON parsing.
	- name: Install apt dependencies
	run: \|
	curl -sSL https://ngrok-agent.s3.amazonaws.com/ngrok.asc \| \
	sudo tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null
	echo "deb https://ngrok-agent.s3.amazonaws.com buster main" \| \
	sudo tee /etc/apt/sources.list.d/ngrok.list

	sudo apt update
	sudo apt install gdb-multiarch ngrok jq

	# We need socat 1.7.4.4 because prior to that there are some issues with
	# setting the baud rate properly. socat is used by the card_client code
	# for various tunnels (e.g. Swan USB port, Swan OpenOCD server, etc.).
	- name: Install socat 1.7.4.4
	run: \|
	wget http://www.dest-unreach.org/socat/download/socat-1.7.4.4.tar.gz
	tar xvf socat-1.7.4.4.tar.gz
	cd socat-1.7.4.4
	./configure
	make -j
	sudo make install

	- name: Install PlatformIO dependencies
	run: \|
	python -m pip install --upgrade pip
	pip install platformio
	cd $PIO_PROJECT_DIR
	pio pkg install -l "Blues Wireless Notecard" -e debug
	# Remove the bundled note-c and put the local working copy there
	NOTE_C_DEP="$GITHUB_WORKSPACE/$PIO_PROJECT_DIR/.pio/libdeps/debug/Blues Wireless Notecard/src/note-c"
	rm -rf "$NOTE_C_DEP"
	mkdir "$NOTE_C_DEP"
	# copy only files in note-c
	find "$GITHUB_WORKSPACE" -maxdepth 1 -type f -exec cp "{}" "${NOTE_C_DEP}" \;

	- name: Start MD5 server
	run: \|
	mkdir md5srv-files
	./scripts/run_md5srv.sh

	- name: Start ngrok
	run: \|
	ngrok config add-authtoken ${{ secrets.NGROK_AUTH_TOKEN }}
	./scripts/run_ngrok.sh

	- name: Check MD5 server is available
	run: \|
	# The request will return a 401 from md5srv, but that's expected without the access token
	# Curl still returns success because it could contact the server
	code=`curl -s -o /dev/null -w "%{http_code}" $MD5SRV_URL`
	if [ "$code" -ge "500" ]; then
	echo "5xx error ($code) from tunnel."
	exit 1
	else
	echo "MD5 server is available."
	fi

	- name: Create Notehub accesss token
	if: env.CREATE_NOTEHUB_ROUTES!='false'
	run: \|
	curl -f -X POST \
	-L 'https://${{ env.NOTEHUB }}/oauth2/token' \
	-H 'content-type: application/x-www-form-urlencoded' \
	-d grant_type=client_credentials \
	-d client_id=${{ secrets.NOTEHUB_HIL_CLIENT_ID }} \
	-d client_secret=${{ secrets.NOTEHUB_HIL_CLIENT_SECRET }} \| \
	{ token=$(jq -r .access_token); echo "NOTEHUB_ACCESS_TOKEN=$token" >> $GITHUB_ENV; }

	- name: Create Notehub HTTP route
	if: env.CREATE_NOTEHUB_ROUTES!='false'
	run: \|
	# ?note=1 instructs the MD5 server to process the content as an event, extracting the path
	# from the event body.
	route_req=`jq -n --arg TOKEN "$MD5SRV_TOKEN" --arg LABEL "$NOTEHUB_HTTP_ROUTE_LABEL" --arg URL "$MD5SRV_URL/?note=1" --argjson TIMEOUT $NOTEHUB_ROUTE_TIMEOUT \
	'{ "label":$LABEL, "type":"http", "http":{ "timeout":$TIMEOUT, "filter": { "type":"include", "files": ["cardbinary.qo"] }, "url":$URL, "http_headers": { "X-Access-Token":$TOKEN } } }'`
	echo "Notehub HTTP route request: $route_req"
	route=`echo "$route_req" \| curl -s -f -X POST -L "https://$NOTEHUB_API/v1/projects/${NOTEHUB_PROJECT_UID}/routes" \
	-H "Authorization: Bearer $NOTEHUB_ACCESS_TOKEN" -d @-`
	echo "Notehub HTTP route: $route"
	route_uid=`echo $route \| jq -r .uid`
	if [ -n "$route_uid" ]; then
	echo "NOTEHUB_HTTP_ROUTE_UID=$route_uid" >> $GITHUB_ENV
	else
	echo "Failed to create or parse Notehub HTTP route."
	exit 1
	fi

	- name: Create Notehub proxy route
	if: env.CREATE_NOTEHUB_ROUTES!='false'
	run: \|
	ALIAS="$NOTEHUB_PROXY_ROUTE_ALIAS"
	route=`jq -n --arg TOKEN "$MD5SRV_TOKEN" --arg LABEL "$NOTEHUB_PROXY_ROUTE_LABEL" --arg URL "$MD5SRV_URL" --arg ALIAS "$ALIAS" --argjson TIMEOUT $NOTEHUB_ROUTE_TIMEOUT \
	'{ "label":$LABEL, "type":"proxy", "proxy":{ "timeout":$TIMEOUT, "url":$URL, "alias":$ALIAS, "http_headers": { "X-Access-Token":$TOKEN } } }' \
	\| curl -s -f -X POST -L "https://api.notefile.net/v1/projects/${NOTEHUB_PROJECT_UID}/routes" \
	-H "Authorization: Bearer $NOTEHUB_ACCESS_TOKEN" -d @-`
	echo "Notehub proxy route: $route"
	route_uid=`echo "$route" \| jq -r .uid`
	if [ -n $route_uid ]; then
	echo "NOTEHUB_PROXY_ROUTE_UID=$route_uid" >> $GITHUB_ENV
	echo "NOTEHUB_PROXY_ROUTE_ALIAS=$ALIAS" >> $GITHUB_ENV
	else
	echo "Failed to create or parse Notehub proxy route."
	exit 1
	fi

	# The notestation CLI uses the notecard CLI.
	- name: Install notecard CLI
	run: \|
	git clone --depth=1 https://github.com/blues/note-cli.git
	cd note-cli/notecard
	go build .
	sudo cp notecard /usr/bin/

	- name: Download and install notestation CLI v2.2.0
	env:
	GH_TOKEN: ${{ secrets.NOTESTATION_RELEASE_DOWNLOAD_TOKEN }}
	run: \|
	api_url="https://api.github.com/repos/blues/notestation/releases/tags/v2.2.0"

	# Query the release asset metadata.
	asset_info=$(curl -s -H "Authorization: token $GH_TOKEN" "$api_url")

	# Extract the release asset URL and name.
	asset_url=$(echo "$asset_info" \| jq -r '.assets[] \| select(.name \| endswith(".whl")) \| .url')
	asset_name=$(echo "$asset_info" \| jq -r '.assets[] \| select(.name \| endswith(".whl")) \| .name')

	if [ -z "$asset_url" ] \|\| [ -z "$asset_name" ]; then
	echo "ERROR: No wheel found in release assets."
	exit 1
	fi

	# Download the wheel using the original filename.
	curl -L -H "Authorization: token $GH_TOKEN" \
	-H "Accept: application/octet-stream" \
	"$asset_url" -o "$asset_name"

	# Create a virtual environment and install the notestation CLI in it.
	python -m venv venv
	source venv/bin/activate
	wheel_file=$(ls *.whl)
	pip install "$wheel_file"

	- name: Build and upload test firmware and run tests
	run: \|
	source venv/bin/activate

	# Reserve the Notestation in the background
	nohup notestation client reserve --notestation barcelona-notestation-1 &> card_client.log &
	PID=$!

	timeout=600 # 10 minutes in seconds
	interval=3 # Check every 3 seconds
	elapsed=0

	# Wait for the reservation directory to appear under ~/.notestation
	RESERVATION_DIR="$HOME/.notestation/pid-${PID}_barcelona-notestation-1"
	while [ ! -d "$RESERVATION_DIR" ]; do
	sleep $interval
	elapsed=$((elapsed + interval))

	# Check if the client process is still running
	if ! kill -0 $PID 2>/dev/null; then
	echo "Error: Notestation client process $PID has terminated unexpectedly."
	echo "$(cat card_client.log)"
	exit 1
	fi

	if [ $elapsed -ge $timeout ]; then
	echo "Timeout reached: Reservation directory $RESERVATION_DIR did not appear."
	kill $PID 2>/dev/null
	echo "$(cat card_client.log)"
	exit 1
	fi
	done

	echo "Notestation reserved."

	# If firmware update is requested, flash it using notestation client flash
	if [ "${{ env.FIRMWARE_S3_URI }}" == "nightly" ]; then
	echo "Flashing nightly firmware..."
	notestation client flash --notecard-nightly
	echo "Firmware flash complete."
	fi

	# Set these environment variables, which are read in platformio.ini in
	# order to flash the Swan with the test firmware.
	RES_FILE="$RESERVATION_DIR/reservation.json"
	export MCU_GDB_SERVER_IP="$(jq -r '.hostname' $RES_FILE)"
	export MCU_GDB_SERVER_PORT="$(jq -r '.debug_servers[] \| select(.target == "host_mcu") \| .gdb_port' $RES_FILE)"
	export GDB_CMD="gdb-multiarch"
	export TEST_PORT="$RESERVATION_DIR/host_mcu_usb"

	if [ -z "$MCU_GDB_SERVER_IP" ] \|\| [ "$MCU_GDB_SERVER_IP" == "null" ]; then
	echo "Error: MCU_GDB_SERVER_IP is empty or not defined."
	kill $PID 2>/dev/null
	exit 1
	fi

	if [ -z "$MCU_GDB_SERVER_PORT" ] \|\| [ "$MCU_GDB_SERVER_PORT" == "null" ]; then
	echo "Error: MCU_GDB_SERVER_PORT is empty or not defined."
	kill $PID 2>/dev/null
	exit 1
	fi

	export PLATFORMIO_BUILD_FLAGS="'-D NOTEHUB_PROXY_ROUTE_ALIAS=\"$NOTEHUB_PROXY_ROUTE_ALIAS\"' '-D PRODUCT_UID=\"$NOTEHUB_PRODUCT_UID\"'"
	echo "build flags $PLATFORMIO_BUILD_FLAGS"
	cd $GITHUB_WORKSPACE/$PIO_PROJECT_DIR

	# Enable Notecard USB trace output.
	printf '{"req":"card.trace","mode":"usb","trace":"req,comm,sync,mem"}\n' > "$RESERVATION_DIR/notecard_usb"

	# Stream Notecard USB output to trace.log in the background.
	socat -u GOPEN:"$RESERVATION_DIR/notecard_usb",rawer - >> "$GITHUB_WORKSPACE/trace.log" &
	TRACE_PID=$!
	echo "Notecard trace capture started (PID: $TRACE_PID)."

	# Run the tests. It's important that we provided --no-reset here. If
	# we don't, PlatformIO tries to fiddle with DTR and RTS, and that
	# causes an exception because the serial port for the Swan isn't a
	# local serial port. It's a virtual serial device hooked up to the
	# Notestation over TCP, and from there it's connected to the actual
	# Swan USB port. Trying to do DTR/RTS on the port causes an ioctl
	# error, because PlatformIO is expecting a genuine serial device that
	# plays nicely with the ioctl stuff it wants to use.
	platformio test -v -e debug \
	--no-reset \
	--json-output-path test.json \
	--junit-output-path test.xml

	kill $TRACE_PID 2>/dev/null
	kill $PID 2>/dev/null

	- name: Publish test report
	uses: mikepenz/action-junit-report@v3
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	if: env.GITHUB_TOKEN && (success() \|\| failure()) # always run even if the previous step fails
	with:
	report_paths: '*/test/hitl/card.binary/test.xml'
	check_name: Notecard Binary HIL Tests
	require_tests: true

	- name: Upload trace log
	uses: actions/upload-artifact@v4
	if: always()
	with:
	name: notecard-trace
	path: trace.log
	if-no-files-found: ignore

	- name: Upload MD5 server log
	uses: actions/upload-artifact@v4
	if: always()
	with:
	name: md5srv-log
	path: md5srv.log
	if-no-files-found: ignore

	- name: Cleanup Notehub proxy route
	if: always()
	run: \|
	if [ "$DELETE_NOTEHUB_ROUTES" == "true" ] && [ -n "$NOTEHUB_PROXY_ROUTE_UID" ]; then
	echo "Deleting Notehub proxy route."
	curl -f -s -X DELETE \
	-L "https://api.notefile.net/v1/projects/$NOTEHUB_PROJECT_UID/routes/$NOTEHUB_PROXY_ROUTE_UID" \
	-H "Authorization: Bearer $NOTEHUB_ACCESS_TOKEN"
	fi

	- name: Cleanup Notehub HTTP route
	if: always()
	run: \|
	if [ "$DELETE_NOTEHUB_ROUTES" == "true" ] && [ -n "$NOTEHUB_HTTP_ROUTE_UID" ]; then
	echo "Deleting Notehub HTTP route."
	curl -f -s -X DELETE \
	-L "https://$NOTEHUB_API/v1/projects/$NOTEHUB_PROJECT_UID/routes/$NOTEHUB_HTTP_ROUTE_UID" \
	-H "Authorization: Bearer $NOTEHUB_ACCESS_TOKEN"
	fi

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

fix: very bad memory leaks in low-memory conditions #976

Workflow file

fix: very bad memory leaks in low-memory conditions #976

Uh oh!

Workflow file for this run