CI/CD Pipeline #129

Workflow file for this run

	name: CI/CD Pipeline

	on:
	push:
	branches: [ main, develop ]
	pull_request:
	branches: [ main ]
	schedule:
	# 每天 UTC 02:00 (北京时间 10:00) 运行夜间测试
	- cron: '0 2 * * *'

	env:
	GO_VERSION: '1.23'

	jobs:
	# 代码质量检查
	code-quality:
	name: Code Quality
	runs-on: ubuntu-latest
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Set up Go
	uses: actions/setup-go@v4
	with:
	go-version: ${{ env.GO_VERSION }}

	- name: Cache Go modules
	uses: actions/cache@v3
	with:
	path: ~/go/pkg/mod
	key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
	restore-keys: \|
	${{ runner.os }}-go-

	- name: Download dependencies
	run: go mod download

	- name: Run go fmt
	run: \|
	if [ "$(gofmt -s -l . \| wc -l)" -gt 0 ]; then
	echo "Code is not formatted properly:"
	gofmt -s -l .
	exit 1
	fi

	- name: Run go vet
	run: go vet ./...

	- name: Install golangci-lint
	run: \|
	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh \| sh -s -- -b $(go env GOPATH)/bin v2.0.0

	- name: Run golangci-lint
	run: $(go env GOPATH)/bin/golangci-lint run

	- name: Check documentation structure
	run: \|
	# 检查文档结构是否完整
	echo "Checking documentation structure..."

	# 检查必需的文档文件
	required_docs=(
	"docs/index.md"
	"docs/api/reference.md"
	"docs/guides/examples.md"
	"docs/guides/testing.md"
	"docs/guides/monitoring.md"
	"docs/development/architecture.md"
	)

	missing_docs=()
	for doc in "${required_docs[@]}"; do
	if [ ! -f "$doc" ]; then
	missing_docs+=("$doc")
	fi
	done

	if [ ${#missing_docs[@]} -ne 0 ]; then
	echo "Missing required documentation files:"
	printf '%s\n' "${missing_docs[@]}"
	exit 1
	fi

	echo "✅ All required documentation files are present"

	- name: Validate Makefile targets
	run: \|
	# 检查 Makefile 中的关键目标
	echo "Validating Makefile targets..."

	required_targets=("test" "docker-all-tests" "monitoring" "ci")
	for target in "${required_targets[@]}"; do
	if ! grep -q "^${target}:" Makefile; then
	echo "❌ Missing Makefile target: $target"
	exit 1
	fi
	done

	echo "✅ All required Makefile targets are present"

	# 单元测试
	unit-tests:
	name: Unit Tests
	runs-on: ubuntu-latest
	needs: code-quality
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Set up Go
	uses: actions/setup-go@v4
	with:
	go-version: ${{ env.GO_VERSION }}

	- name: Cache Go modules
	uses: actions/cache@v3
	with:
	path: ~/go/pkg/mod
	key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
	restore-keys: \|
	${{ runner.os }}-go-

	- name: Download dependencies
	run: go mod download

	- name: Run unit tests
	run: \|
	go test -v -race -coverprofile=coverage.out ./...
	go tool cover -func=coverage.out

	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v3
	with:
	file: ./coverage.out
	flags: unittests
	name: codecov-umbrella

	# SQLite 集成测试
	integration-sqlite:
	name: SQLite Integration Tests
	runs-on: ubuntu-latest
	needs: unit-tests
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v3

	- name: Build SQLite test image
	run: \|
	docker compose -f docker-compose.ci.yml build sqlite sqlite-test --no-cache

	- name: Run SQLite integration tests
	run: \|
	# 使用较短的测试时间以适应 CI 环境
	export TEST_DURATION=300s # 5分钟测试
	export CONCURRENT_WORKERS=2
	export RECORDS_PER_WORKER=1000
	export BATCH_SIZE=100
	export BUFFER_SIZE=2000
	export FLUSH_INTERVAL=100ms

	# 运行测试并捕获退出码
	docker compose -f docker-compose.ci.yml up sqlite sqlite-test --abort-on-container-exit --exit-code-from sqlite-test \|\| true

	- name: Prepare SQLite test reports
	run: \|
	mkdir -p ./test/reports-sqlite
	if [ -d ./test/reports ] && [ "$(ls -A ./test/reports)" ]; then
	cp -R ./test/reports/* ./test/reports-sqlite/ \|\| true
	else
	echo "No reports found in ./test/reports"
	fi

	- name: Upload SQLite test reports
	uses: actions/upload-artifact@v4
	if: always()
	with:
	name: sqlite-test-reports
	path: test/reports-sqlite/
	retention-days: 30

	# MySQL 集成测试
	integration-mysql:
	name: MySQL Integration Tests
	runs-on: ubuntu-latest
	needs: unit-tests
	services:
	mysql:
	image: mysql:8.0
	env:
	MYSQL_ROOT_PASSWORD: rootpass123
	MYSQL_DATABASE: batchflow_test
	MYSQL_USER: testuser
	MYSQL_PASSWORD: testpass123
	ports:
	- 3306:3306
	options: >-
	--health-cmd="mysqladmin ping"
	--health-interval=10s
	--health-timeout=5s
	--health-retries=3

	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v3

	- name: Wait for MySQL
	run: \|
	for i in {30..0}; do
	if mysqladmin ping -h 127.0.0.1 -u testuser -ptestpass123 &> /dev/null; then
	break
	fi
	echo 'MySQL is unavailable - sleeping'
	sleep 2
	done

	- name: Build MySQL test image
	run: \|
	docker compose -f docker-compose.ci.yml build mysql mysql-test --no-cache

	- name: Run MySQL integration tests
	run: \|
	# 使用较短的测试时间以适应 CI 环境
	export TEST_DURATION=300s # 5分钟测试
	export CONCURRENT_WORKERS=5
	export RECORDS_PER_WORKER=2000
	export BATCH_SIZE=200
	export BUFFER_SIZE=5000
	export FLUSH_INTERVAL=100ms

	# 更新 MySQL DSN 以连接到 GitHub Actions 的 MySQL 服务
	export MYSQL_DSN="testuser:testpass123@tcp(host.docker.internal:3306)/batchflow_test?parseTime=true&multiStatements=true"

	docker compose -f docker-compose.ci.yml up mysql mysql-test --abort-on-container-exit --exit-code-from mysql-test

	- name: Prepare MySQL test reports
	run: \|
	mkdir -p ./test/reports-mysql
	if [ -d ./test/reports ] && [ "$(ls -A ./test/reports)" ]; then
	cp -R ./test/reports/* ./test/reports-mysql/ \|\| true
	else
	echo "No reports found in ./test/reports"
	fi

	- name: Upload MySQL test reports
	uses: actions/upload-artifact@v4
	if: always()
	with:
	name: mysql-test-reports
	path: test/reports-mysql/
	retention-days: 30

	# PostgreSQL 集成测试
	integration-postgresql:
	name: PostgreSQL Integration Tests
	runs-on: ubuntu-latest
	needs: unit-tests
	services:
	postgres:
	image: postgres:16-alpine
	env:
	POSTGRES_DB: batchflow_test
	POSTGRES_USER: testuser
	POSTGRES_PASSWORD: testpass123
	ports:
	- 5432:5432
	options: >-
	--health-cmd pg_isready
	--health-interval 10s
	--health-timeout 5s
	--health-retries 5

	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v3

	- name: Build PostgreSQL test image
	run: \|
	docker compose -f docker-compose.ci.yml build postgres postgres-test --no-cache

	- name: Run PostgreSQL integration tests
	run: \|
	# 使用较短的测试时间以适应 CI 环境
	export TEST_DURATION=300s # 5分钟测试
	export CONCURRENT_WORKERS=5
	export RECORDS_PER_WORKER=2000
	export BATCH_SIZE=200
	export BUFFER_SIZE=5000
	export FLUSH_INTERVAL=100ms

	# 更新 PostgreSQL DSN 以连接到 GitHub Actions 的 PostgreSQL 服务
	export POSTGRES_DSN="postgres://testuser:testpass123@host.docker.internal:5432/batchflow_test?sslmode=disable"

	docker compose -f docker-compose.ci.yml up postgres postgres-test --abort-on-container-exit --exit-code-from postgres-test

	- name: Prepare PostgreSQL test reports
	run: \|
	mkdir -p ./test/reports-postgres
	if [ -d ./test/reports ] && [ "$(ls -A ./test/reports)" ]; then
	cp -R ./test/reports/* ./test/reports-postgres/ \|\| true
	else
	echo "No reports found in ./test/reports"
	fi

	- name: Upload PostgreSQL test reports
	uses: actions/upload-artifact@v4
	if: always()
	with:
	name: postgresql-test-reports
	path: test/reports-postgres/
	retention-days: 30

	# Redis 集成测试
	integration-redis:
	name: Redis Integration Tests
	runs-on: ubuntu-latest
	needs: unit-tests
	services:
	redis:
	image: redis:7-alpine
	ports:
	- 6379:6379
	options: >-
	--health-cmd "redis-cli ping"
	--health-interval 10s
	--health-timeout 5s
	--health-retries 5

	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v3

	- name: Build Redis test image
	run: \|
	docker compose -f docker-compose.ci.yml build redis redis-test --no-cache

	- name: Run Redis integration tests
	run: \|
	# 使用较短的测试时间以适应 CI 环境
	export TEST_DURATION=300s # 5分钟测试
	export CONCURRENT_WORKERS=5
	export RECORDS_PER_WORKER=2000
	export BATCH_SIZE=200
	export BUFFER_SIZE=5000
	export FLUSH_INTERVAL=100ms

	# 更新 Redis DSN 以连接到 GitHub Actions 的 Redis 服务
	export REDIS_ADDR="host.docker.internal:6379"

	docker compose -f docker-compose.ci.yml up redis redis-test --abort-on-container-exit --exit-code-from redis-test

	- name: Prepare Redis test reports
	run: \|
	mkdir -p ./test/reports-redis
	if [ -d ./test/reports ] && [ "$(ls -A ./test/reports)" ]; then
	cp -R ./test/reports/* ./test/reports-redis/ \|\| true
	else
	echo "No reports found in ./test/reports"
	fi

	- name: Upload Redis test reports
	uses: actions/upload-artifact@v4
	if: always()
	with:
	name: redis-test-reports
	path: test/reports-redis/
	retention-days: 30



	# 性能基准测试（仅在主分支和定时任务中运行）
	performance-benchmarks:
	name: Performance Benchmarks
	runs-on: ubuntu-latest
	needs: [integration-sqlite, integration-mysql, integration-postgresql]
	if: github.ref == 'refs/heads/main' \|\| github.event_name == 'schedule'
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Set up Go
	uses: actions/setup-go@v4
	with:
	go-version: ${{ env.GO_VERSION }}

	- name: Run Go benchmarks
	run: \|
	set -o pipefail
	go test -run=^$ -bench=. -benchmem -count=3 ./... \| tee benchmark_results.txt

	- name: Upload benchmark results
	uses: actions/upload-artifact@v4
	with:
	name: benchmark-results
	path: benchmark_results.txt
	retention-days: 90

	# 生成测试报告汇总
	test-summary:
	name: Test Summary
	runs-on: ubuntu-latest
	needs: [integration-sqlite, integration-mysql, integration-postgresql, integration-redis]
	if: always()
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Download all test reports
	uses: actions/download-artifact@v4
	with:
	path: ./all-reports

	- name: Generate test summary
	run: \|
	echo "# BatchFlow CI/CD Test Summary" > test_summary.md
	echo "" >> test_summary.md
	echo "## Test Results Overview" >> test_summary.md
	echo "" >> test_summary.md

	# 检查各个测试的状态
	if [ "${{ needs.integration-sqlite.result }}" == "success" ]; then
	echo "✅ SQLite Integration Tests: PASSED" >> test_summary.md
	else
	echo "❌ SQLite Integration Tests: FAILED (Expected - SQLite architectural limitations)" >> test_summary.md
	fi

	if [ "${{ needs.integration-mysql.result }}" == "success" ]; then
	echo "✅ MySQL Integration Tests: PASSED" >> test_summary.md
	else
	echo "❌ MySQL Integration Tests: FAILED" >> test_summary.md
	fi

	if [ "${{ needs.integration-postgresql.result }}" == "success" ]; then
	echo "✅ PostgreSQL Integration Tests: PASSED" >> test_summary.md
	else
	echo "❌ PostgreSQL Integration Tests: FAILED" >> test_summary.md
	fi

	if [ "${{ needs.integration-redis.result }}" == "success" ]; then
	echo "✅ Redis Integration Tests: PASSED" >> test_summary.md
	else
	echo "❌ Redis Integration Tests: FAILED" >> test_summary.md
	fi



	echo "" >> test_summary.md
	echo "## Test Coverage Summary" >> test_summary.md
	echo "- Database Support: MySQL ✅ \| PostgreSQL ✅ \| SQLite ⚠️ \| Redis ✅" >> test_summary.md

	echo "- Documentation: Structure validation ✅" >> test_summary.md
	echo "" >> test_summary.md
	echo "## Notes" >> test_summary.md
	echo "- SQLite test failures are expected due to architectural limitations" >> test_summary.md
	echo "- MySQL, PostgreSQL, and Redis should pass all tests" >> test_summary.md

	echo "- Detailed reports are available in the artifacts" >> test_summary.md

	- name: Upload test summary
	uses: actions/upload-artifact@v4
	with:
	name: test-summary
	path: test_summary.md
	retention-days: 30

	- name: Comment PR with test results
	if: github.event_name == 'pull_request'
	uses: actions/github-script@v6
	with:
	script: \|
	const fs = require('fs');
	const summary = fs.readFileSync('test_summary.md', 'utf8');

	github.rest.issues.createComment({
	issue_number: context.issue.number,
	owner: context.repo.owner,
	repo: context.repo.repo,
	body: summary
	});

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

CI/CD Pipeline #129

Workflow file

CI/CD Pipeline #129

Uh oh!

Workflow file for this run