Kubernetes for rustcrawl

.dockerignore

@@ -0,0 +1,15 @@
+# Keep the build context small and reproducible.
+target/
+.git/
+.github/
+deploy/
+.rustcrawl/
+
+# Crawl artifacts and local output.
+*.jsonl
+
+# Editor / OS noise.
+.idea/
+.vscode/
+*.swp
+.DS_Store

.github/workflows/docker.yml

@@ -0,0 +1,71 @@
+name: docker
+
+# Build and publish the container image to GitHub Container Registry (GHCR).
+#
+# Images are pushed on:
+#   - pushes to main          -> ghcr.io/<owner>/rustcrawl:edge (+ :sha)
+#   - semver tags (vX.Y.Z)    -> ghcr.io/<owner>/rustcrawl:X.Y.Z, :X.Y, :latest
+# Pull requests build only (no push) to validate the Dockerfile.
+on:
+  push:
+    branches: [main]
+    tags: ["v*.*.*"]
+  pull_request:
+    paths:
+      - "Dockerfile"
+      - ".dockerignore"
+      - "crates/**"
+      - "Cargo.toml"
+      - "Cargo.lock"
+      - ".github/workflows/docker.yml"
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GHCR
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=sha
+            type=raw,value=edge,enable={{is_default_branch}}
+            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

Dockerfile

@@ -0,0 +1,47 @@
+# syntax=docker/dockerfile:1
+#
+# Multi-stage build for the `rustcrawl` CLI.
+#
+# The image is intentionally headless: with no TTY attached the CLI skips the
+# Crawl Deck dashboard, streams one JSON object per page to stdout, and prints a
+# one-line summary to stderr. That makes it a clean fit for Kubernetes Jobs and
+# CronJobs where logs are collected by the cluster.
+#
+# Build:   docker build -t rustcrawl:local .
+# Run:     docker run --rm rustcrawl:local https://example.com -n 100 --no-save -q
+
+# ---- Build stage -----------------------------------------------------------
+FROM rust:1-bookworm AS builder
+
+WORKDIR /app
+
+# Copy the full workspace. BuildKit cache mounts keep the dependency registry
+# and target directory warm across builds, so iterative builds stay fast
+# without fragile dummy-source caching tricks.
+COPY . .
+
+RUN --mount=type=cache,target=/usr/local/cargo/registry \
+    --mount=type=cache,target=/app/target \
+    cargo build --release -p rustcrawl-cli \
+    && cp target/release/rustcrawl /usr/local/bin/rustcrawl
+
+# ---- Runtime stage ---------------------------------------------------------
+# reqwest is built against the system TLS stack (native-tls/OpenSSL), so the
+# runtime needs libssl plus root certificates for HTTPS verification.
+FROM debian:bookworm-slim AS runtime
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends ca-certificates libssl3 \
+    && rm -rf /var/lib/apt/lists/*
+
+# Run as an unprivileged, well-known UID so the manifests can assert
+# runAsNonRoot without surprises.
+RUN useradd --system --create-home --uid 10001 --user-group crawler
+
+COPY --from=builder /usr/local/bin/rustcrawl /usr/local/bin/rustcrawl
+
+USER crawler
+WORKDIR /home/crawler
+
+ENTRYPOINT ["rustcrawl"]
+CMD ["--help"]

deploy/helm/rustcrawl/.helmignore

@@ -0,0 +1,8 @@
+.DS_Store
+.git/
+.gitignore
+*.tmproj
+*.bak
+*.swp
+.idea/
+.vscode/

deploy/helm/rustcrawl/Chart.yaml

@@ -0,0 +1,18 @@
+apiVersion: v2
+name: rustcrawl
+description: A fast, efficient web crawler that runs as a Kubernetes Job or CronJob.
+type: application
+# Chart version: bump on chart changes.
+version: 0.1.0
+# Tracks the rustcrawl release this chart targets; used as the default image tag.
+appVersion: "0.1.0"
+keywords:
+  - crawler
+  - spider
+  - scraper
+  - web
+home: https://github.com/rustcrawl/rustcrawl
+sources:
+  - https://github.com/rustcrawl/rustcrawl
+maintainers:
+  - name: rustcrawl contributors

deploy/helm/rustcrawl/templates/NOTES.txt

@@ -0,0 +1,39 @@
+rustcrawl has been installed as release "{{ .Release.Name }}".
+
+Mode: {{ .Values.mode }}
+Image: {{ include "rustcrawl.image" . }}
+
+{{- if eq .Values.mode "job" }}
+
+A one-off crawl Job was created. Watch it with:
+
+  kubectl get job {{ include "rustcrawl.fullname" . }} -n {{ .Release.Namespace }}
+  kubectl logs -f job/{{ include "rustcrawl.fullname" . }} -n {{ .Release.Namespace }}
+
+Page output is JSON Lines on stdout (visible in the logs above).
+{{- else if eq .Values.mode "cronjob" }}
+
+A recurring crawl CronJob was created with schedule "{{ .Values.schedule }}".
+
+  kubectl get cronjob {{ include "rustcrawl.fullname" . }} -n {{ .Release.Namespace }}
+
+Trigger an immediate run for testing:
+
+  kubectl create job --from=cronjob/{{ include "rustcrawl.fullname" . }} {{ include "rustcrawl.fullname" . }}-manual -n {{ .Release.Namespace }}
+{{- end }}
+
+{{- if .Values.persistence.enabled }}
+
+Persistence is enabled at {{ .Values.persistence.mountPath }}. Make sure your
+crawl args write there, for example:
+
+  crawl:
+    extraArgs: ["-o", "{{ .Values.persistence.mountPath }}/pages.jsonl"]
+{{- else }}
+
+Output goes to stdout (collected by your cluster logging). To persist results,
+set persistence.enabled=true and add an output flag, e.g.:
+
+  --set persistence.enabled=true \
+  --set 'crawl.extraArgs={-o,/data/pages.jsonl}'
+{{- end }}

deploy/helm/rustcrawl/templates/_helpers.tpl

@@ -0,0 +1,124 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "rustcrawl.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Fully qualified app name.
+*/}}
+{{- define "rustcrawl.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name (include "rustcrawl.name" .) | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Common labels.
+*/}}
+{{- define "rustcrawl.labels" -}}
+helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{ include "rustcrawl.selectorLabels" . }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}
+
+{{/*
+Selector labels.
+*/}}
+{{- define "rustcrawl.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "rustcrawl.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end -}}
+
+{{/*
+Container image reference.
+*/}}
+{{- define "rustcrawl.image" -}}
+{{- printf "%s:%s" .Values.image.repository (.Values.image.tag | default .Chart.AppVersion) -}}
+{{- end -}}
+
+{{/*
+The crawler command line: seeds, then args, then extra args, then the flags we
+always force in-cluster (`--no-save` keeps the root filesystem read-only, and
+`-q` runs headless without the terminal dashboard).
+*/}}
+{{- define "rustcrawl.args" -}}
+{{- range .Values.crawl.seeds }}
+- {{ . | quote }}
+{{- end }}
+{{- range .Values.crawl.args }}
+- {{ . | quote }}
+{{- end }}
+{{- range .Values.crawl.extraArgs }}
+- {{ . | quote }}
+{{- end }}
+- "--no-save"
+- "-q"
+{{- end -}}
+
+{{/*
+The shared Job spec (also embedded inside the CronJob's jobTemplate). Callers
+must include this with the correct nindent for their nesting depth.
+*/}}
+{{- define "rustcrawl.jobSpec" -}}
+backoffLimit: {{ .Values.job.backoffLimit }}
+activeDeadlineSeconds: {{ .Values.job.activeDeadlineSeconds }}
+ttlSecondsAfterFinished: {{ .Values.job.ttlSecondsAfterFinished }}
+template:
+  metadata:
+    labels:
+      {{- include "rustcrawl.selectorLabels" . | nindent 6 }}
+  spec:
+    restartPolicy: {{ .Values.job.restartPolicy }}
+    {{- with .Values.imagePullSecrets }}
+    imagePullSecrets:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+    securityContext:
+      {{- toYaml .Values.podSecurityContext | nindent 6 }}
+    containers:
+      - name: {{ .Chart.Name }}
+        image: {{ include "rustcrawl.image" . | quote }}
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        args:
+          {{- include "rustcrawl.args" . | nindent 10 }}
+        {{- with .Values.env }}
+        env:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        securityContext:
+          {{- toYaml .Values.securityContext | nindent 10 }}
+        resources:
+          {{- toYaml .Values.resources | nindent 10 }}
+        volumeMounts:
+          - name: tmp
+            mountPath: /tmp
+          {{- if .Values.persistence.enabled }}
+          - name: data
+            mountPath: {{ .Values.persistence.mountPath }}
+          {{- end }}
+    volumes:
+      - name: tmp
+        emptyDir: {}
+      {{- if .Values.persistence.enabled }}
+      - name: data
+        persistentVolumeClaim:
+          claimName: {{ .Values.persistence.existingClaim | default (printf "%s-data" (include "rustcrawl.fullname" .)) }}
+      {{- end }}
+    {{- with .Values.nodeSelector }}
+    nodeSelector:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+    {{- with .Values.affinity }}
+    affinity:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+    {{- with .Values.tolerations }}
+    tolerations:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+{{- end -}}

deploy/helm/rustcrawl/templates/cronjob.yaml

@@ -0,0 +1,17 @@
+{{- if eq .Values.mode "cronjob" }}
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ include "rustcrawl.fullname" . }}
+  labels:
+    {{- include "rustcrawl.labels" . | nindent 4 }}
+spec:
+  schedule: {{ .Values.schedule | quote }}
+  concurrencyPolicy: {{ .Values.cron.concurrencyPolicy }}
+  successfulJobsHistoryLimit: {{ .Values.cron.successfulJobsHistoryLimit }}
+  failedJobsHistoryLimit: {{ .Values.cron.failedJobsHistoryLimit }}
+  startingDeadlineSeconds: {{ .Values.cron.startingDeadlineSeconds }}
+  jobTemplate:
+    spec:
+      {{- include "rustcrawl.jobSpec" . | nindent 6 }}
+{{- end }}

deploy/helm/rustcrawl/templates/job.yaml

@@ -0,0 +1,10 @@
+{{- if eq .Values.mode "job" }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "rustcrawl.fullname" . }}
+  labels:
+    {{- include "rustcrawl.labels" . | nindent 4 }}
+spec:
+  {{- include "rustcrawl.jobSpec" . | nindent 2 }}
+{{- end }}

deploy/helm/rustcrawl/templates/pvc.yaml

@@ -0,0 +1,17 @@
+{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ printf "%s-data" (include "rustcrawl.fullname" .) }}
+  labels:
+    {{- include "rustcrawl.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    {{- toYaml .Values.persistence.accessModes | nindent 4 }}
+  resources:
+    requests:
+      storage: {{ .Values.persistence.size | quote }}
+  {{- if .Values.persistence.storageClassName }}
+  storageClassName: {{ .Values.persistence.storageClassName | quote }}
+  {{- end }}
+{{- end }}